diff --git a/.agents/skills/apple-container/SKILL.md b/.agents/skills/apple-container/SKILL.md new file mode 100644 index 00000000000..fed7ce1e260 --- /dev/null +++ b/.agents/skills/apple-container/SKILL.md @@ -0,0 +1,63 @@ +--- +name: apple-container +description: Run RustPython tests in Apple's `container` CLI Linux environment. Use when user asks to run or compare Linux test results on macOS. +allowed-tools: Bash(container:*) Bash(cargo:*) Read Grep Glob +--- + +# Run Tests in Linux Container (Apple `container` CLI) + +Run RustPython tests inside a Linux container using Apple's `container` CLI. +NEVER use Docker, Podman, or any other container runtime. Only use the `container` command. + +## Arguments + +- Test command to run (examples: `test_io`, `test_codecs -v`, `test_io -v -m "test_errors"`) + +## Prerequisites + +- The `container` CLI is installed via `brew install container`. +- The dev image `rustpython-dev` is already built. + +## Workflow + +1. Check whether the test container is already running: + + ```shell + container list 2>/dev/null | grep rustpython-test + ``` + +2. Start the container if it is not running: + + ```shell + container run -d --name rustpython-test -m 8G -c 4 \ + --mount type=bind,source="$(pwd)",target=/workspace \ + -w /workspace rustpython-dev sleep infinity + ``` + +3. Run the requested test command inside the container: + + ```shell + container exec rustpython-test cargo run --release -- -m test + ``` + +4. Report results: + +- Show pass/fail summary and expected failures / unexpected successes. +- Highlight new failures compared to macOS results, if available. +- Do not stop or remove the container after testing. + +## Notes + +- The workspace is bind-mounted, so local code changes are immediately available in the container. +- Use `container exec rustpython-test sh -c "..."` for any command inside the container. +- Rebuild after code changes with: + + ```shell + container exec rustpython-test sh -c "cargo build --release" + ``` + +- Stop the container when explicitly requested: + + ```shell + container rm -f rustpython-test + ``` diff --git a/.agents/skills/investigate-test-failure/SKILL.md b/.agents/skills/investigate-test-failure/SKILL.md new file mode 100644 index 00000000000..e63dfeeec1b --- /dev/null +++ b/.agents/skills/investigate-test-failure/SKILL.md @@ -0,0 +1,53 @@ +--- +name: investigate-test-failure +description: Investigate a failing RustPython test, compare with CPython behavior, and decide whether to implement a fix or prepare incompatibility issue details. +allowed-tools: Bash(python3:*) Bash(cargo run:*) Bash(gh issue create:*) Read Grep Glob Bash(git add:*) Bash(git commit:*) Bash(cargo fmt:*) Bash(git diff:*) Task +--- + +# Investigate Test Failure + +Investigate why a specific test is failing and determine whether it can be fixed now or should be reported as an incompatibility issue. + +## Arguments + +- Failed test identifier (example: `test_inspect.TestGetSourceBase.test_getsource_reload`) + +## Workflow + +1. Analyze the failure cause: + +- Read the test code. +- Analyze failure message and traceback. +- Check related RustPython implementation code. + +2. Verify behavior in CPython: + +- Run the test with `python3 -m unittest`. +- Document expected behavior/output. + +3. Determine fix feasibility: + +- Simple fix (import issues, small logic bugs): implement fix, run formatting and checks, review changes, and commit. +- Complex fix (major missing feature): gather issue information and report to user. + +Pre-commit review process: + +- Run `git diff` to review local changes. +- Use a focused subagent review to compare implementation against CPython behavior and check for missed edge cases. +- Commit only after review passes. + +4. For complex issues, collect issue information using `.github/ISSUE_TEMPLATE/report-incompatibility.md`: + +- Feature: missing or broken Python feature. +- Minimal reproduction code. +- CPython behavior (`python3`). +- RustPython behavior (`cargo run`). +- Python documentation reference link. + +Report the collected information to the user. Create an issue only when explicitly requested. + +Example issue command: + +```shell +gh issue create --template report-incompatibility.md --title "..." --body "..." +``` diff --git a/.agents/skills/rustpython-capi-expansion/SKILL.md b/.agents/skills/rustpython-capi-expansion/SKILL.md new file mode 100644 index 00000000000..5b56929a825 --- /dev/null +++ b/.agents/skills/rustpython-capi-expansion/SKILL.md @@ -0,0 +1,60 @@ +--- +name: rustpython-capi-expansion +description: Implement missing RustPython C-API functions in crates/capi using the pyo3-ffi header split mapping (`pyo3-ffi/src/*.rs`, mirroring CPython C API headers). Use this whenever the user asks to add or port C-API functions (for example from setobject.h, dictobject.h, unicodeobject.h) or add capi tests. +--- + +# RustPython C-API Expansion + +Use this workflow for adding missing C-API functions to RustPython. + +## Source of truth for target files + +- Use this mapping source: `pyo3-ffi/src/*.rs`, which mirrors the CPython header split used by the C API. +- Map requested header APIs to `crates/capi/src/.rs` using that split. Examples: + - `setobject.h` -> `crates/capi/src/setobject.rs` + - `dictobject.h` -> `crates/capi/src/dictobject.rs` + - `unicodeobject.h` -> `crates/capi/src/unicodeobject.rs` +- Do not invent alternate target modules when the header split implies a direct target. +- If the target file is not present yet, create it and wire it in `crates/capi/src/lib.rs`. + +## Implementation workflow + +1. Identify requested missing APIs from the user request and their originating C API header. +2. Open nearby capi modules in `crates/capi/src/` and follow existing style and patterns. +3. Implement only the requested functions in the mapped target file. +4. Keep behavior aligned with CPython C-API contracts. +5. Prefer using existing `rustpython-vm` functionality as much as possible instead of re-implementing behavior in capi. +6. If a needed `rustpython-vm` helper exists but is private, make it public with a minimal, focused visibility change. +7. Prefer direct contract assumptions over defensive null checks unless required by the established local style. +8. Add basic tests only; do not overfit with very specific edge-case clutter. +9. In tests, use `pyo3` only as a safe wrapper over the API. Avoid raw pointer-heavy direct FFI-style tests. +10. Run tests from `crates/capi`. + +## Testing rules + +- Run test commands with working directory set to `crates/capi`. +- Prefer targeted tests first (module/function filter), then broader capi tests if needed. +- Keep test names concise (no required `test_` prefix). + +## Style rules + +- Follow existing RustPython capi coding style in neighboring files. +- Reuse `rustpython-vm` methods and types first; avoid duplicating VM logic in capi wrappers. +- When exposing previously private VM helpers, keep the API surface minimal and avoid unrelated refactors. +- Only expose and implement ABI-stable C-API surface needed for `abi3` / `abi3t`. +- Add comments only when they explain non-obvious behavior. +- Keep edits minimal and focused on requested API expansion. + +## Completion checklist + +- [ ] All requested functions implemented in mapped target file. +- [ ] New module exported in `crates/capi/src/lib.rs` when applicable. +- [ ] Basic safe-wrapper `pyo3` tests added/updated. +- [ ] Tests executed from `crates/capi` and passing for changed area. +- [ ] Final response includes changed file paths and test command summary. + +## Example prompts this skill should handle + +- "Implement these missing functions from `dictobject.h`." +- "Add `setobject.h` C-API functions in RustPython and include basic tests." +- "Port the listed `unicodeobject.h` APIs in capi, follow existing style, and run tests from `crates/capi`." diff --git a/.agents/skills/upgrade-pylib-next/SKILL.md b/.agents/skills/upgrade-pylib-next/SKILL.md new file mode 100644 index 00000000000..e68e7316509 --- /dev/null +++ b/.agents/skills/upgrade-pylib-next/SKILL.md @@ -0,0 +1,34 @@ +--- +name: upgrade-pylib-next +description: Pick the next CPython stdlib module ready for upgrade based on update_lib todo output and skip modules with open upgrade PRs. +allowed-tools: Skill(upgrade-pylib) Bash(gh pr list:*) Bash(cargo run:*) +--- + +# Upgrade Next Python Library + +Find the next Python library module ready for upgrade and then run the `upgrade-pylib` workflow for it. + +## Workflow + +1. Get current TODO status: + +```shell +cargo run --release -- scripts/update_lib todo 2>/dev/null +``` + +2. Get open upgrade PRs: + +```shell +gh pr list --search "Update in:title" --json number,title --template '{{range .}}#{{.number}} {{.title}}{{"\\n"}}{{end}}' +``` + +3. From TODO output, select modules in this priority order: + +- `[ ] [no deps]` +- `[ ] [0/n]` where dependencies are all upgraded (for example `[0/3]`, `[0/5]`) + +4. Skip modules that already have an open upgrade PR. + +5. Run upgrade for selected module using the `upgrade-pylib` workflow. + +If no modules match, report that eligible modules are blocked by dependencies. diff --git a/.agents/skills/upgrade-pylib/SKILL.md b/.agents/skills/upgrade-pylib/SKILL.md new file mode 100644 index 00000000000..84400e5b7a2 --- /dev/null +++ b/.agents/skills/upgrade-pylib/SKILL.md @@ -0,0 +1,102 @@ +--- +name: upgrade-pylib +description: Upgrade a Python standard library module from CPython into RustPython using scripts/update_lib and then triage and mark remaining failures. +allowed-tools: Bash(git add:*) Bash(git commit:*) Bash(git diff:*) Bash(cargo run:*) Bash(python3 scripts/update_lib quick:*) Bash(python3 scripts/update_lib auto-mark:*) +--- + +# Upgrade Python Library from CPython + +Upgrade a Python standard library module from CPython to RustPython. + +## Arguments + +- Library name or path (examples: `inspect`, `asyncio`, `json`, `cpython/Lib/inspect.py`, `cpython/Lib/json/`) + +## Important: Report Tool Issues First + +If `scripts/update_lib` shows a bug, missing automation, or unexpected behavior, stop the upgrade and report the tooling issue first, including: + +1. What you were trying to do: + +- Library name. +- Full command used. + +2. What went wrong or what is missing. +3. Expected vs actual behavior. + +## Workflow + +1. Run quick upgrade with update_lib: + +- `python3 scripts/update_lib quick ` +- or `python3 scripts/update_lib quick cpython/Lib/.py` +- or `python3 scripts/update_lib quick cpython/Lib//` + +This step copies library files, patches tests while preserving markers where possible, runs tests, auto-marks new failures, removes stale `@unittest.expectedFailure`, and creates a commit. + +If warnings like `WARNING: TestCFoo does not exist in remote file` appear, class structure changed and some markers must be restored manually. + +2. Review diff and restore RustPython-specific changes: + +- Run `git diff Lib/test/test_`. +- Restore only changes with explicit `RUSTPYTHON` comments. +- Do not restore unrelated upstream CPython changes. + +3. Investigate failing dependent tests: + +- Get dependencies: + + ```shell + cargo run --release -- scripts/update_lib deps + ``` + +- Find direct dependent test modules from the `- [ ] :` line. +- Run dependent tests and collect failures: + + ```shell + cargo run --release -- -m test 2>&1 | grep -E "^(FAIL|ERROR):" + ``` + +- For each failing test identifier, investigate with the `investigate-test-failure` workflow. + +4. Mark remaining failures with auto-mark: + +- `python3 scripts/update_lib auto-mark Lib/test/test_.py --mark-failure` +- or `python3 scripts/update_lib auto-mark Lib/test/test_/ --mark-failure` + +Note: `--mark-failure` marks all current failures including regressions; review carefully. + +5. Handle panics manually: + +- For tests that panic/crash/hang, use `@unittest.skip("TODO: RUSTPYTHON; ...")` rather than expectedFailure. + +6. Handle class-specific failures: + +- If only one subclass fails (for example `TestCFoo` but not `TestPyFoo`), move marker to the failing subclass override. + +7. Commit test-fix markers as a separate commit: + +```shell +git add -u && git commit -m "Mark failing tests" +``` + +## Example usage + +```text +upgrade-pylib inspect +upgrade-pylib json +upgrade-pylib asyncio +upgrade-pylib cpython/Lib/inspect.py +upgrade-pylib cpython/Lib/json/ +``` + +## Notes + +- The `cpython/` directory should contain the CPython source used for sync. +- `scripts/update_lib` modes: + - `quick`: patch + auto-mark + - `migrate`: patch only + - `auto-mark`: mark failures only + - `copy-lib`: copy library files only +- Patching transfers `@unittest.expectedFailure` and `@unittest.skip` with `TODO: RUSTPYTHON` markers where possible. +- The tool does not preserve every RustPython-specific change; always review and restore explicit RUSTPYTHON-marked logic. diff --git a/.cargo/config.toml b/.cargo/config.toml index 635229119f8..4e54d1d3b18 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -6,3 +6,16 @@ rustflags = "-C link-args=-Wl,--stack,8000000" [target.wasm32-unknown-unknown] rustflags = ["--cfg=getrandom_backend=\"wasm_js\""] + +# Enforce a 1 MB stack limit for WASI targets. +# (Runaway heap growth is fixed by the smaller DataStack chunk +# and the size‑optimized wasm‑release profile.) +[target.wasm32-wasip1] +rustflags = [ + "-C", "link-arg=-zstack-size=1048576", +] + +[target.wasm32-wasip2] +rustflags = [ + "-C", "link-arg=-zstack-size=1048576", +] diff --git a/.claude/commands/investigate-test-failure.md b/.claude/commands/investigate-test-failure.md deleted file mode 100644 index e9d6b2d2d2c..00000000000 --- a/.claude/commands/investigate-test-failure.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -allowed-tools: Bash(python3:*), Bash(cargo run:*), Read, Grep, Glob, Bash(git add:*), Bash(git commit:*), Bash(cargo fmt:*), Bash(git diff:*), Task ---- - -# Investigate Test Failure - -Investigate why a specific test is failing and determine if it can be fixed or needs an issue. - -## Arguments -- `$ARGUMENTS`: Failed test identifier (e.g., `test_inspect.TestGetSourceBase.test_getsource_reload`) - -## Steps - -1. **Analyze failure cause** - - Read the test code - - Analyze failure message/traceback - - Check related RustPython code - -2. **Verify behavior in CPython** - - Run the test with `python3 -m unittest` to confirm expected behavior - - Document the expected output - -3. **Determine fix feasibility** - - **Simple fix** (import issues, small logic bugs): Fix code → Run `cargo fmt --all` → Pre-commit review → Commit - - **Complex fix** (major unimplemented features): Collect issue info and report to user - - **Pre-commit review process**: - - Run `git diff` to see the changes - - Use Task tool with `general-purpose` subagent to review: - - Compare implementation against cpython/ source code - - Verify the fix aligns with CPython behavior - - Check for any missed edge cases - - Proceed to commit only after review passes - -4. **For complex issues - Collect issue information** - Following `.github/ISSUE_TEMPLATE/report-incompatibility.md` format: - - - **Feature**: Description of missing/broken Python feature - - **Minimal reproduction code**: Smallest code that reproduces the issue - - **CPython behavior**: Result when running with python3 - - **RustPython behavior**: Result when running with cargo run - - **Python Documentation link**: Link to relevant CPython docs - - Report collected information to the user. Issue creation is done only upon user request. - - Example issue creation command: - ``` - gh issue create --template report-incompatibility.md --title "..." --body "..." - ``` diff --git a/.claude/commands/upgrade-pylib-next.md b/.claude/commands/upgrade-pylib-next.md deleted file mode 100644 index 712b79433b3..00000000000 --- a/.claude/commands/upgrade-pylib-next.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -allowed-tools: Skill(upgrade-pylib), Bash(gh pr list:*) ---- - -# Upgrade Next Python Library - -Find the next Python library module ready for upgrade and run `/upgrade-pylib` for it. - -## Current TODO Status - -!`cargo run --release -- scripts/update_lib todo 2>/dev/null` - -## Open Upgrade PRs - -!`gh pr list --search "Update in:title" --json number,title --template '{{range .}}#{{.number}} {{.title}}{{"\n"}}{{end}}'` - -## Instructions - -From the TODO list above, find modules matching these patterns (in priority order): - -1. `[ ] [no deps]` - Modules with no dependencies (can be upgraded immediately) -2. `[ ] [0/n]` - Modules where all dependencies are already upgraded (e.g., `[0/3]`, `[0/5]`) - -These patterns indicate modules that are ready to upgrade without blocking dependencies. - -**Important**: Skip any modules that already have an open PR in the "Open Upgrade PRs" list above. - -**After identifying a suitable module**, run: -``` -/upgrade-pylib -``` - -If no modules match these criteria, inform the user that all eligible modules have dependencies that need to be upgraded first. diff --git a/.claude/commands/upgrade-pylib.md b/.claude/commands/upgrade-pylib.md deleted file mode 100644 index d54305d2616..00000000000 --- a/.claude/commands/upgrade-pylib.md +++ /dev/null @@ -1,157 +0,0 @@ ---- -allowed-tools: Bash(git add:*), Bash(git commit:*), Bash(python3 scripts/update_lib quick:*), Bash(python3 scripts/update_lib auto-mark:*) ---- - -# Upgrade Python Library from CPython - -Upgrade a Python standard library module from CPython to RustPython. - -## Arguments -- `$ARGUMENTS`: Library name to upgrade (e.g., `inspect`, `asyncio`, `json`) - -## Important: Report Tool Issues First - -If during the upgrade process you encounter any of the following issues with `scripts/update_lib`: -- A feature that should be automated but isn't supported -- A bug or unexpected behavior in the tool -- Missing functionality that would make the upgrade easier - -**STOP the upgrade and report the issue first.** Describe: -1. What you were trying to do - - Library name - - The full command executed (e.g. python scripts/update_lib quick cpython/Lib/$ARGUMENTS.py) -2. What went wrong or what's missing -3. Expected vs actual behavior - -This helps improve the tooling for future upgrades. - -## Steps - -1. **Run quick upgrade with update_lib** - - Run: `python3 scripts/update_lib quick $ARGUMENTS` (module name) - - Or: `python3 scripts/update_lib quick cpython/Lib/$ARGUMENTS.py` (library file path) - - Or: `python3 scripts/update_lib quick cpython/Lib/$ARGUMENTS/` (library directory path) - - This will: - - Copy library files (delete existing `Lib/$ARGUMENTS.py` or `Lib/$ARGUMENTS/`, then copy from `cpython/Lib/`) - - Patch test files preserving existing RustPython markers - - Run tests and auto-mark new test failures (not regressions) - - Remove `@unittest.expectedFailure` from tests that now pass - - Create a git commit with the changes - - **Handle warnings**: If you see warnings like `WARNING: TestCFoo does not exist in remote file`, it means the class structure changed and markers couldn't be transferred automatically. These need to be manually restored in step 2 or added in step 3. - -2. **Review git diff and restore RUSTPYTHON-specific changes** - - Run `git diff Lib/test/test_$ARGUMENTS` to review all changes - - **Only restore changes that have explicit `RUSTPYTHON` comments**. Look for: - - `# XXX: RUSTPYTHON` or `# XXX RUSTPYTHON` - Comments marking RustPython-specific code modifications - - `# TODO: RUSTPYTHON` - Comments marking tests that need work - - Code changes with inline `# ... RUSTPYTHON` comments - - **Do NOT restore other diff changes** - these are likely upstream CPython changes, not RustPython-specific modifications - - When restoring, preserve the original context and formatting - -3. **Investigate test failures with subagent** - - First, get dependent tests using the deps command: - ``` - cargo run --release -- scripts/update_lib deps $ARGUMENTS - ``` - - Look for the line `- [ ] $ARGUMENTS: test_xxx test_yyy ...` to get the direct dependent tests - - Run those tests to collect failures: - ``` - cargo run --release -- -m test test_xxx test_yyy ... 2>&1 | grep -E "^(FAIL|ERROR):" - ``` - - For example, if deps output shows `- [ ] linecache: test_bdb test_inspect test_linecache test_traceback test_zipimport`, run: - ``` - cargo run --release -- -m test test_bdb test_inspect test_linecache test_traceback test_zipimport 2>&1 | grep -E "^(FAIL|ERROR):" - ``` - - For each failure, use the Task tool with `general-purpose` subagent to investigate: - - Subagent should follow the `/investigate-test-failure` skill workflow - - Pass the failed test identifier as the argument (e.g., `test_inspect.TestGetSourceBase.test_getsource_reload`) - - If subagent can fix the issue easily: fix and commit - - If complex issue: subagent collects issue info and reports back (issue creation on user request only) - - Using subagent prevents context pollution in the main conversation - -4. **Mark remaining test failures with auto-mark** - - Run: `python3 scripts/update_lib auto-mark Lib/test/test_$ARGUMENTS.py --mark-failure` - - Or for directory: `python3 scripts/update_lib auto-mark Lib/test/test_$ARGUMENTS/ --mark-failure` - - This will: - - Run tests and mark ALL failing tests with `@unittest.expectedFailure` - - Remove `@unittest.expectedFailure` from tests that now pass - - **Note**: The `--mark-failure` flag marks all failures including regressions. Review the changes before committing. - -5. **Handle panics manually** - - If any tests cause panics/crashes (not just assertion failures), they need `@unittest.skip` instead: - ```python - @unittest.skip("TODO: RUSTPYTHON; panics with 'index out of bounds'") - def test_crashes(self): - ... - ``` - - auto-mark cannot detect panics automatically - check the test output for crash messages - -6. **Handle class-specific failures** - - If a test fails only in the C implementation (TestCFoo) but passes in the Python implementation (TestPyFoo), or vice versa, move the marker to the specific subclass: - ```python - # Base class - no marker here - class TestFoo: - def test_something(self): - ... - - class TestPyFoo(TestFoo, PyTest): pass - - class TestCFoo(TestFoo, CTest): - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_something(self): - return super().test_something() - ``` - -7. **Commit the test fixes** - - Run: `git add -u && git commit -m "Mark failing tests"` - - This creates a separate commit for the test markers added in steps 2-6 - -## Example Usage -``` -# Using module names (recommended) -/upgrade-pylib inspect -/upgrade-pylib json -/upgrade-pylib asyncio - -# Using library paths (alternative) -/upgrade-pylib cpython/Lib/inspect.py -/upgrade-pylib cpython/Lib/json/ -``` - -## Example: Restoring RUSTPYTHON changes - -When git diff shows removed RUSTPYTHON-specific code like: -```diff --# XXX RUSTPYTHON: we don't import _json as fresh since... --cjson = import_helper.import_fresh_module('json') #, fresh=['_json']) -+cjson = import_helper.import_fresh_module('json', fresh=['_json']) -``` - -You should restore the RustPython version: -```python -# XXX RUSTPYTHON: we don't import _json as fresh since... -cjson = import_helper.import_fresh_module('json') #, fresh=['_json']) -``` - -## Notes -- The cpython/ directory should contain the CPython source that we're syncing from -- `scripts/update_lib` package handles patching and auto-marking: - - `quick` - Combined patch + auto-mark (recommended) - - `migrate` - Only migrate (patch), no test running - - `auto-mark` - Only run tests and mark failures - - `copy-lib` - Copy library files (not tests) -- The patching: - - Transfers `@unittest.expectedFailure` and `@unittest.skip` decorators with `TODO: RUSTPYTHON` markers - - Adds `import unittest # XXX: RUSTPYTHON` if needed for the decorators - - **Limitation**: If a class was restructured (e.g., method overrides removed), update_lib will warn and skip those markers -- The smart auto-mark: - - Marks NEW test failures automatically (tests that didn't exist before) - - Does NOT mark regressions (existing tests that now fail) - these are warnings - - Removes `@unittest.expectedFailure` from tests that now pass -- The script does NOT preserve all RustPython-specific changes - you must review `git diff` and restore them -- Common RustPython markers to look for: - - `# XXX: RUSTPYTHON` or `# XXX RUSTPYTHON` - Inline comments for code modifications - - `# TODO: RUSTPYTHON` - Test skip/failure markers - - Any code with `RUSTPYTHON` in comments that was removed in the diff -- **Important**: Not all changes in the git diff need to be restored. Only restore changes that have explicit `RUSTPYTHON` comments. Other changes are upstream CPython updates. diff --git a/.claude/scripts/setup-env.sh b/.claude/scripts/setup-env.sh new file mode 100755 index 00000000000..f5d28a3f14c --- /dev/null +++ b/.claude/scripts/setup-env.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Claude Code web session startup script +# Sets up the development environment for RustPython + +set -e + +cd /home/user/RustPython + +echo "=== RustPython dev environment setup ===" + +# 1. Ensure python3 points to 3.13+ (needed for scripts/update_lib) +# /usr/local/bin takes precedence over /usr/bin in PATH, +# so we update the symlink there directly. +CURRENT_PY=$(python3 --version 2>&1 | grep -oP '\d+\.\d+') +if [ "$(printf '%s\n' "3.13" "$CURRENT_PY" | sort -V | head -1)" != "3.13" ]; then + echo "Upgrading python3 default to 3.13..." + # Find best available Python >= 3.13 + TARGET="" + for ver in python3.14 python3.13; do + if command -v "$ver" &>/dev/null; then + TARGET=$(command -v "$ver") + break + fi + done + if [ -n "$TARGET" ]; then + # Override /usr/local/bin/python3 if it exists and is outdated + if [ -e /usr/local/bin/python3 ]; then + sudo ln -sf "$TARGET" /usr/local/bin/python3 + fi + # Also set /usr/bin via update-alternatives + sudo update-alternatives --install /usr/bin/python3 python3 "$TARGET" 3 2>/dev/null || true + sudo update-alternatives --set python3 "$TARGET" 2>/dev/null || true + echo "python3 now: $(python3 --version)" + else + echo "WARNING: No Python 3.13+ found. scripts/update_lib may not work." + fi +else + echo "python3 already >= 3.13: $(python3 --version)" +fi + +# 2. Clone CPython source if not present (needed for scripts/update_lib) +if [ ! -d "cpython" ]; then + echo "Cloning CPython v3.14.3 (shallow)..." + git clone --depth 1 --branch v3.14.3 https://github.com/python/cpython.git cpython + echo "CPython source ready." +else + echo "CPython source already present." +fi + +echo "=== Setup complete ===" diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index 0e5d1cce2d9..81274276e2c 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -4,13 +4,19 @@ argdefs argtypes asdl asname +ASNATIVEBYTES +atopen +atext attro augassign badcert badsyntax baseinfo basetype +binop +bltin boolop +BUFMAX BUILDSTDLIB bxor byteswap @@ -25,41 +31,65 @@ carg cellarg cellvar cellvars +ceval cfield +cfws +CFWS +chans +cidx +cjkcodecs CLASSDEREF classdict cmpop +CNOTAB codedepth CODEUNIT +CONIN +CONOUT +constevaluator +consti CONVFUNC convparam copyslot cpucount +datastack defaultdict denom +deopt +deopts dictbytype DICTFLAG dictoffset distpoint dynload elts +eooh eofs +EOOH +esize evalloop excepthandler exceptiontable +fastlocal +fastlocals fblock fblocks fdescr +fdst ffi_argtypes fielddesc fieldlist fileutils finalbody +finalizers +firsttraceable flowgraph formatfloat +freelist freevar freevars fromlist +fsrc getdict getfunc getiter @@ -72,54 +102,87 @@ HASPOINTER HASSTRUCT HASUNION heaptype +hexdigit HIGHRES +ialloc IFUNC IMMUTABLETYPE INCREF inlinedepth inplace +inpos +ioffset +isbytecode +ishidden ismine ISPOINTER +isoctal iteminfo Itertool +iused keeped kwnames kwonlyarg kwonlyargs +kwonlydefaults lasti libffi linearise +lineful lineiterator linetable +LNOTAB loadfast localsplus +localspluskinds Lshift +lslpp lsprof MAXBLOCKS +MAXFSTRINGLEVEL maxdepth +mbiencoder +mbstreamreader +mbstreamwriter metavars miscompiles mult multibytecodec +mystricmp +mystrnicmp +nameobj nameop +nargsf +nblocks +ncells +ncellsused +ncellvars nconsts newargs newfree NEWLOCALS newsemlockobject +nextop nfrees nkwargs nkwelts +nlocalsplus +nointerrupt +noffsets Nondescriptor noninteger nops noraise +npools nseen NSIGNALS numer +nvars +obmalloc opname opnames orelse +osmodule outparam outparm paramfunc @@ -127,51 +190,77 @@ parg pathconfig patma peepholer +pegen phcount platstdlib +ploc posonlyarg posonlyargs prec +preds preinitialized pybuilddir pycore +pyinner pydecimal +pyerrors +pyframe Pyfunc +pylifecycle pymain +pymem pyrepl +pystate +pystrcmp PYTHONTRACEMALLOC +PYTHONUTF8 pythonw PYTHREAD_NAME +recvd releasebuffer repr resinfo +retarget Rshift SA_ONSTACK saveall scls setdict setfunc +setprofileallthreads SETREF setresult setslice +settraceallthreads +sget SLOTDEFINED +SMALLBUF SOABI SSLEOF stackdepth +stackref staticbase stginfo storefast stringlib +stringized structseq subkwargs subparams subscr sval swappedbytes +swaptimize +sysdict +tbexc +tbstderr templatelib +testconsole +threadstate ticketer tmptype tok_oldval +tstate tvars typeobject typeparam @@ -179,10 +268,13 @@ Typeparam typeparams typeslots unaryop +unboundop uncollectable Unhandle unparse unparser +untargeted +untracking VARKEYWORDS varkwarg venvlauncher @@ -194,7 +286,12 @@ wbits weakreflist weakrefobject webpki +winconsoleio withitem withs +worklist +XSETREF +xibufferview xstat XXPRIME +ENOTCAPABLE diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt index 2dd31f8f579..7ea660a5d1f 100644 --- a/.cspell.dict/python-more.txt +++ b/.cspell.dict/python-more.txt @@ -1,8 +1,10 @@ abiflags abstractmethods +addcompare aenter aexit aiter +altzone anext anextawaitable annotationlib @@ -24,6 +26,7 @@ breakpointhook cformat chunksize classcell +classmethods closefd closesocket codepoint @@ -32,6 +35,8 @@ codesize contextvar cpython cratio +ctype +ctypes dealloc debugbuild decompressor @@ -62,6 +67,7 @@ fillchar fillvalue finallyhandler firstiter +fobj firstlineno fnctl frombytes @@ -74,6 +80,8 @@ fstring fstrings ftruncate genexpr +genexpressions +getargs getattro getcodesize getdefaultencoding @@ -83,14 +91,17 @@ getformat getframe getframemodulename getnewargs +getopt getpip getrandom getrecursionlimit getrefcount getsizeof getswitchinterval +getweakref getweakrefcount getweakrefs +getweakrefs getwindowsversion gmtoff groupdict @@ -101,10 +112,16 @@ idfunc idiv idxs impls +infd indexgroup infj +inittab +Inittab instancecheck instanceof +instrs +interpchannels +interpqueues irepeat isabstractmethod isbytes @@ -129,6 +146,7 @@ listcomp longrange lvalue mappingproxy +markupbase maskpri maxdigits MAXGROUPS @@ -144,6 +162,7 @@ mformat mro mros multiarch +mymodule namereplace nanj nbytes @@ -156,8 +175,10 @@ nlocals NOARGS nonbytes Nonprintable +onceregistry origname ospath +outfd pendingcr phello platlibdir @@ -168,9 +189,14 @@ posonlyargcount prepending profilefunc pycache +pycapsule pycodecs pycs +pydatetime pyexpat +PYGILSTATE +pyio +pymain PYTHONAPI PYTHONBREAKPOINT PYTHONDEBUG @@ -220,10 +246,13 @@ scproxy seennl setattro setcomp +setprofileallthreads setrecursionlimit setswitchinterval +settraceallthreads showwarnmsg signum +sitebuiltins slotnames STACKLESS stacklevel @@ -232,14 +261,17 @@ startpos subclassable subclasscheck subclasshook +subclassing suboffset suboffsets SUBPATTERN +subpatterns sumprod surrogateescape surrogatepass sysconf sysconfigdata +sysdict sysvars teedata thisclass @@ -266,35 +298,10 @@ warnopts weaklist weakproxy weakrefs +weakrefset winver withdata xmlcharrefreplace xoptions xopts yieldfrom -addcompare -altzone -classmethods -ctype -ctypes -genexpressions -getargs -getopt -getweakref -getweakrefs -inittab -Inittab -interpchannels -interpqueues -markupbase -mymodule -pydatetime -pyio -pymain -setprofileallthreads -settraceallthreads -sitebuiltins -subclassing -subpatterns -sysdict -weakrefset diff --git a/.cspell.dict/rust-more.txt b/.cspell.dict/rust-more.txt index c3ebd61833a..b53639c3b41 100644 --- a/.cspell.dict/rust-more.txt +++ b/.cspell.dict/rust-more.txt @@ -5,7 +5,9 @@ biguint bindgen bitand bitflags +bitflagset bitor +bitvec bitxor bstr byteorder @@ -48,6 +50,7 @@ modpow msvc muldiv nanos +noalias nonoverlapping objclass peekable @@ -58,6 +61,7 @@ powi prepended punct replacen +retag rmatch rposition rsplitn @@ -89,5 +93,3 @@ widestring winapi winresource winsock -bitvec -Bitvec diff --git a/.cspell.dict/rustpython.txt b/.cspell.dict/rustpython.txt new file mode 100644 index 00000000000..07099bbb171 --- /dev/null +++ b/.cspell.dict/rustpython.txt @@ -0,0 +1,33 @@ +cfgs +miri +py +pyarg +pyargs +pyast +pyattr +pyclass +pyclassmethod +pyexception +pyfunction +pygetset +pyimpl +pylib +pymath +pymethod +pymodule +pyname +pyobj +pyobject +pypayload +pyref +pyslot +pystaticmethod +pystone +pystr +pystruct +pystructseq +pytype +qsbr +rustix +struc +zelf diff --git a/.cspell.json b/.cspell.json index ebed8664e58..433fbc7f9fb 100644 --- a/.cspell.json +++ b/.cspell.json @@ -9,6 +9,7 @@ "@cspell/dict-win32/cspell-ext.json", "@cspell/dict-shell/cspell-ext.json", ], + "allowCompoundWords": true, // language - current active spelling language "language": "en", // dictionaries - list of the names of the dictionaries to use @@ -16,6 +17,7 @@ "cpython", // Sometimes keeping same terms with cpython is easy "python-more", // Python API terms not listed in python "rust-more", // Rust API terms not listed in rust + "rustpython", // RustPython derive macros and internal terms "en_US", "softwareTerms", "c", @@ -38,116 +40,98 @@ { "name": "rust-more", "path": "./.cspell.dict/rust-more.txt" + }, + { + "name": "rustpython", + "path": "./.cspell.dict/rustpython.txt" } ], "ignorePaths": [ "**/__pycache__/**", "target/**", - "Lib/**" + "Lib/**", + "crates/host_env/**" + ], + "overrides": [ + { + "filename": "crates/common/src/encodings/cjk/**", + "words": [ + "CHOSEONG", + "EUCKR", + "famille", + "GBCOMMON", + "jong", + "JONGSEONG", + "JUNGSEONG", + "PHINT", + "Xabc" + ] + } ], // words - list of words to be always considered correct + // (compound words like pyarg, baseclass, microbenchmark are handled by allowCompoundWords) "words": [ - "RUSTPYTHONPATH", - // RustPython terms + "AESGCM", "aiterable", "alnum", - "baseclass", - "boxvec", - "Bytecode", - "cfgs", - "codegen", + "COMPLEMENTOFALL", + "COMPLEMENTOFDEFAULT", + "csock", "coro", + "contig", + "Crnl", "dedentations", "dedents", "deduped", - "downcastable", - "downcasted", - "dumpable", + "deoptimized", + "deoptimize", + "EECDH", "emscripten", "excs", - "finalizer", - "GetSet", - "groupref", - "internable", + "fdigits", + "feff", + "flufl", + "fnfe", + "fsdefault", + "ifexp", + "implicits", + "inity", "interps", "jitted", "jitting", + "kwonly", "lossily", - "makeunicodedata", - "microbenchmark", - "microbenchmarks", - "miri", - "notrace", + "mcache", + "MLKEM", "oparg", - "openat", - "pyarg", - "pyargs", - "pyast", - "PyAttr", + "opargs", + "pointee", "pyc", - "PyClass", - "PyClassMethod", - "PyException", - "PyFunction", - "pygetset", - "pyimpl", - "pylib", - "pymath", - "pymember", - "PyMethod", - "PyModule", - "pyname", - "pyobj", - "PyObject", - "pypayload", - "PyProperty", - "pyref", - "PyResult", - "pyslot", - "PyStaticMethod", - "pystone", - "pystr", - "pystruct", - "pystructseq", - "pytrace", - "pytype", - "reducelib", - "richcompare", - "RustPython", + "qmark", + "reborrow", + "reborrows", + "reparenting", + "reraises", + "reraising", "significand", - "struc", - "summands", // plural of summand - "sysmodule", - "tracebacks", - "typealiases", - "unhashable", - "uninit", + "SUITEB", + "summands", + "TESTFN", + "TZPATH", "unraisable", - "unresizable", - "varint", "wasi", - "zelf", + "weaked", // unix "posixshmem", "shm", "CLOEXEC", - "codeset", "endgrent", "gethrvtime", "getrusage", - "nanosleep", "sigaction", - "sighandler", "WRLCK", // win32 - "birthtime", - "IFEXEC", - // "stat" - "FIRMLINK", - // CPython internal names - "sysdict", - "settraceallthreads", - "setprofileallthreads" + "IFEXEC" ], // flagWords - list of words to be always considered incorrect "flagWords": [ diff --git a/.gitattributes b/.gitattributes index 71f807e8ed6..44fc4c8f9b0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,10 +1,14 @@ Lib/** linguist-vendored Cargo.lock linguist-generated *.snap linguist-generated -merge -vm/src/stdlib/ast/gen.rs linguist-generated -merge Lib/*.py text working-tree-encoding=UTF-8 eol=LF **/*.rs text working-tree-encoding=UTF-8 eol=LF -crates/rustpython_doc_db/src/*.inc.rs linguist-generated=true +crates/vm/src/stdlib/ast/gen.rs linguist-generated -merge +crates/doc/src/data.inc.rs generated +crates/compiler-core/src/bytecode/opcode_metadata.rs generated +crates/common/src/encodings/cjk/mappings_*.rs generated + +.github/workflows/*.lock.yml linguist-generated=true merge=ours # Binary data types *.aif binary @@ -45,10 +49,11 @@ Lib/venv/scripts/posix/* text eol=lf # Language aware diff headers # https://tekin.co.uk/2020/10/better-git-diff-output-for-ruby-python-elixir-and-more # https://gist.github.com/tekin/12500956bd56784728e490d8cef9cb81 -*.css diff=css -*.html diff=html -*.py diff=python -*.md diff=markdown +*.css diff=css +*.html diff=html +*.py diff=python +*.md diff=markdown +*.inc.rs diff=rust # Generated files # https://github.com/github/linguist/blob/master/docs/overrides.md @@ -58,11 +63,11 @@ Lib/venv/scripts/posix/* text eol=lf # [attr]generated linguist-generated=true diff=generated -Lib/_opcode_metadata.py generated -Lib/keyword.py generated -Lib/idlelib/help.html generated -Lib/test/certdata/*.pem generated -Lib/test/certdata/*.0 generated -Lib/test/levenshtein_examples.json generated -Lib/test/test_stable_abi_ctypes.py generated -Lib/token.py generated +Lib/_opcode_metadata.py generated +Lib/keyword.py generated +Lib/idlelib/help.html generated +Lib/test/certdata/*.pem generated +Lib/test/certdata/*.0 generated +Lib/test/levenshtein_examples.json generated +Lib/test/test_stable_abi_ctypes.py generated +Lib/token.py generated diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000000..51c6dd747a8 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,13 @@ + + +- [ ] Closes #xxxx + +One of checkbox below must be checked. +- [ ] I did not use AI to write the code of this patch. +- [ ] This PR follows our [AI policy](https://github.com/RustPython/.github/blob/main/AI_POLICY.md) + +## Summary + + diff --git a/.github/actions/install-linux-deps/action.yml b/.github/actions/install-linux-deps/action.yml new file mode 100644 index 00000000000..6ce8393ce41 --- /dev/null +++ b/.github/actions/install-linux-deps/action.yml @@ -0,0 +1,94 @@ +# This action installs a few dependencies necessary to build RustPython on Linux. +# It can be configured depending on which libraries are needed: +# +# ``` +# - uses: ./.github/actions/install-linux-deps +# with: +# gcc-multilib: true +# musl-tools: false +# ``` +# +# See the `inputs` section for all options and their defaults. Note that you must checkout the +# repository before you can use this action. +# +# This action will only install dependencies when the current operating system is Linux. It will do +# nothing on any other OS (macOS, Windows). + +name: Install Linux dependencies +description: Installs the dependencies necessary to build RustPython on Linux. +inputs: + gcc-multilib: + description: Install gcc-multilib (gcc-multilib) + required: false + default: "false" + musl-tools: + description: Install musl-tools (musl-tools) + required: false + default: "false" + gcc-aarch64-linux-gnu: + description: Install gcc-aarch64-linux-gnu (gcc-aarch64-linux-gnu) + required: false + default: "false" + gcc-mingw-w64-x86-64: + description: Install gcc-mingw-w64-x86-64 (gcc-mingw-w64-x86-64) + required: false + default: "false" + clang: + description: Install clang (clang) + required: false + default: "false" +runs: + using: composite + steps: + - name: Install Linux dependencies + shell: bash + if: ${{ runner.os == 'Linux' }} + env: + GCC_MULTILIB: ${{ inputs.gcc-multilib }} + MUSL_TOOLS: ${{ inputs.musl-tools }} + CLANG: ${{ inputs.clang }} + GCC_AARCH64_LINUX_GNU: ${{ inputs.gcc-aarch64-linux-gnu }} + GCC_MINGW_W64_X86_64: ${{ inputs.gcc-mingw-w64-x86-64 }} + run: | + # `apt-get update` has no deadline of its own, so a source that takes + # the connection and then stops answering holds the job rather than + # failing it, and the retry below never runs. Bound each attempt and + # give the transports a timeout to fail on. + apt_update() { + sudo timeout 300 apt-get \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=20 \ + -o Acquire::https::Timeout=20 \ + update + } + + if ! apt_update; then + echo "::warning::apt-get update did not finish; disabling nonessential Microsoft apt sources and retrying" + for source in /etc/apt/sources.list.d/*microsoft* /etc/apt/sources.list.d/*azure-cli*; do + if [ -e "$source" ]; then + sudo mv "$source" "$source.disabled" + fi + done + apt_update + fi + + packages=() + if [[ "$GCC_MULTILIB" == "true" ]]; then + packages+=(gcc-multilib) + fi + if [[ "$MUSL_TOOLS" == "true" ]]; then + packages+=(musl-tools) + fi + if [[ "$CLANG" == "true" ]]; then + packages+=(clang) + fi + if [[ "$GCC_AARCH64_LINUX_GNU" == "true" ]]; then + packages+=(gcc-aarch64-linux-gnu linux-libc-dev-arm64-cross libc6-dev-arm64-cross) + fi + if [[ "$GCC_MINGW_W64_X86_64" == "true" ]]; then + packages+=(gcc-mingw-w64-x86-64) + fi + + if ((${#packages[@]})); then + sudo apt-get install --no-install-recommends "${packages[@]}" + fi diff --git a/.github/actions/install-macos-deps/action.yml b/.github/actions/install-macos-deps/action.yml new file mode 100644 index 00000000000..46abef197a4 --- /dev/null +++ b/.github/actions/install-macos-deps/action.yml @@ -0,0 +1,47 @@ +# This action installs a few dependencies necessary to build RustPython on macOS. By default it installs +# autoconf, automake and libtool, but can be configured depending on which libraries are needed: +# +# ``` +# - uses: ./.github/actions/install-macos-deps +# with: +# openssl: true +# libtool: false +# ``` +# +# See the `inputs` section for all options and their defaults. Note that you must checkout the +# repository before you can use this action. +# +# This action will only install dependencies when the current operating system is macOS. It will do +# nothing on any other OS (Linux, Windows). + +name: Install macOS dependencies +description: Installs the dependencies necessary to build RustPython on macOS. +inputs: + autoconf: + description: Install autoconf (autoconf) + required: false + default: "true" + automake: + description: Install automake (automake) + required: false + default: "true" + libtool: + description: Install libtool (libtool) + required: false + default: "true" + openssl: + description: Install openssl (openssl@3) + required: false + default: "false" +runs: + using: composite + steps: + - name: Install macOS dependencies + shell: bash + if: ${{ runner.os == 'macOS' }} + run: > + brew install + ${{ fromJSON(inputs.autoconf) && 'autoconf' || '' }} + ${{ fromJSON(inputs.automake) && 'automake' || '' }} + ${{ fromJSON(inputs.libtool) && 'libtool' || '' }} + ${{ fromJSON(inputs.openssl) && 'openssl@3' || '' }} diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json new file mode 100644 index 00000000000..ad986cbd051 --- /dev/null +++ b/.github/aw/actions-lock.json @@ -0,0 +1,14 @@ +{ + "entries": { + "actions/github-script@v8": { + "repo": "actions/github-script", + "version": "v8", + "sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd" + }, + "github/gh-aw/actions/setup@v0.43.22": { + "repo": "github/gh-aw/actions/setup", + "version": "v0.43.22", + "sha": "fe858c3e14589bf396594a0b106e634d9065823e" + } + } +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b3b7b446e4a..e9e2041d482 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,15 +1,177 @@ +# cspell:ignore manyhow tinyvec zeroize version: 2 updates: - package-ecosystem: cargo - directory: / + directories: + - "/" + - "crates/*" schedule: interval: weekly + cooldown: + default-days: 7 + semver-major-days: 30 + semver-minor-days: 7 + semver-patch-days: 3 + groups: + criterion: + patterns: + - "criterion*" + crypto: + patterns: + - "digest" + - "md-5" + - "sha-1" + - "sha1" + - "sha2" + - "sha3" + - "shake" + - "blake2" + - "hmac" + - "pbkdf2" + futures: + patterns: + - "futures*" + get-size2: + patterns: + - "get-size*2" + iana-time-zone: + patterns: + - "iana-time-zone*" + jiff: + patterns: + - "jiff*" + lexical: + patterns: + - "lexical*" + libffi: + patterns: + - "libffi*" + malachite: + patterns: + - "malachite*" + manyhow: + patterns: + - "manyhow*" + num: + patterns: + - "num-bigint" + - "num-complex" + - "num-integer" + - "num-iter" + - "num-rational" + - "num-traits" + num_enum: + patterns: + - "num_enum*" + openssl: + patterns: + - "openssl*" + parking_lot: + patterns: + - "parking_lot*" + phf: + patterns: + - "phf*" + plotters: + patterns: + - "plotters*" + portable-atomic: + patterns: + - "portable-atomic*" + pyo3: + patterns: + - "pyo3*" + quote-use: + patterns: + - "quote-use*" + random: + patterns: + - "getrandom" + - "mt19937" + - "rand*" + - "chacha20" + rayon: + patterns: + - "rayon*" + regex: + patterns: + - "regex*" + result-like: + patterns: + - "result-like*" + security-framework: + patterns: + - "security-framework*" + serde: + patterns: + - "serde" + - "serde_core" + - "serde_derive" + system-configuration: + patterns: + - "system-configuration*" + thiserror: + patterns: + - "thiserror*" + time: + patterns: + - "time*" + tinyvec: + patterns: + - "tinyvec*" + tls_codec: + patterns: + - "tls_codec*" + toml: + patterns: + - "toml*" + unix: + patterns: + - "mac_address" + - "nix" + - "rustyline" + wasm-bindgen: + patterns: + - "wasm-bindgen*" + wasmtime: + patterns: + - "cranelift*" + - "wasmtime*" + webpki-root: + patterns: + - "webpki-root*" + windows: + patterns: + - "windows*" + zerocopy: + patterns: + - "zerocopy*" + zeroize: + patterns: + - "zeroize*" ignore: # TODO: Remove when we use ruff from crates.io # for some reason dependabot only updates the Cargo.lock file when dealing # with git dependencies. i.e. not updating the version in Cargo.toml - - dependency-name: "ruff_*" + - dependency-name: "ruff_*" - package-ecosystem: github-actions directory: / schedule: interval: weekly + cooldown: + default-days: 7 + - package-ecosystem: npm + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + semver-major-days: 30 + semver-minor-days: 7 + semver-patch-days: 3 + - package-ecosystem: pre-commit + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6ca14c351f5..4e94b4ea629 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,121 +8,81 @@ on: name: CI +permissions: + contents: read + # Cancel previous workflows if they are the same workflow on same ref (branch/tags) # with the same event (push/pull_request) even they are in progress. # This setting will help reduce the number of duplicated workflows. concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true env: - CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls - CARGO_ARGS_NO_SSL: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite + CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls-aws-lc,host_env + CARGO_ARGS_NO_SSL: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,host_env # Crates excluded from workspace builds: # - rustpython_wasm: requires wasm target - # - rustpython-compiler-source: deprecated # - rustpython-venvlauncher: Windows-only - WORKSPACE_EXCLUDES: --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher - # Skip additional tests on Windows. They are checked on Linux and MacOS. - # test_glob: many failing tests - # test_pathlib: panic by surrogate chars - # test_posixpath: OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)') - # test_venv: couple of failing tests - WINDOWS_SKIPS: >- - test_asyncio - test_glob - test_rlcompleter - test_pathlib - test_posixpath - test_venv - # PLATFORM_INDEPENDENT_TESTS are tests that do not depend on the underlying OS. They are currently - # only run on Linux to speed up the CI. - PLATFORM_INDEPENDENT_TESTS: >- - test__colorize - test_array - test_asyncgen - test_binop - test_bisect - test_bool - test_bytes - test_call - test_class - test_cmath - test_collections - test_complex - test_contains - test_copy - test_dataclasses - test_decimal - test_decorators - test_defaultdict - test_deque - test_dict - test_dictcomps - test_dictviews - test_dis - test_enumerate - test_exception_variations - test_float - test_format - test_fractions - test_genericalias - test_genericclass - test_grammar - test_range - test_index - test_int - test_int_literal - test_isinstance - test_iter - test_iterlen - test_itertools - test_json - test_keyword - test_keywordonlyarg - test_list - test_long - test_longexp - test_math - test_operator - test_ordered_dict - test_pep646_syntax - test_pow - test_raise - test_richcmp - test_scope - test_set - test_slice - test_sort - test_string - test_string_literals - test_strtod - test_structseq - test_subclassinit - test_super - test_syntax - test_tstring - test_tuple - test_types - test_unary - test_unpack - test_weakref - test_yield_from - ENV_POLLUTING_TESTS_COMMON: >- - ENV_POLLUTING_TESTS_LINUX: >- - ENV_POLLUTING_TESTS_MACOS: >- - ENV_POLLUTING_TESTS_WINDOWS: >- - # Python version targeted by the CI. - PYTHON_VERSION: "3.14.2" + WORKSPACE_EXCLUDES: --exclude rustpython_wasm --exclude rustpython-venvlauncher X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include + CARGO_INCREMENTAL: 0 + CARGO_PROFILE_TEST_DEBUG: 0 + CARGO_PROFILE_DEV_DEBUG: 0 + CARGO_PROFILE_RELEASE_DEBUG: 0 + CARGO_TERM_COLOR: always + CI: true + FORCE_COLOR: 1 jobs: + determine_changes: + name: Determine changes + runs-on: ubuntu-slim + outputs: + # Flag that is raised when any rust code is changed. + rust_code: ${{ steps.check_rust_code.outputs.changed }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Determine merge base + id: merge_base + run: | + sha=$(git merge-base HEAD "origin/${BASE_REF}") + echo "sha=${sha}" >> "$GITHUB_OUTPUT" + env: + BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }} + + - name: Check if there was any code related change + id: check_rust_code + run: | + if git diff --quiet "${MERGE_BASE}...HEAD" -- \ + ':Cargo.toml' \ + ':Cargo.lock' \ + ':rust-toolchain.toml' \ + ':.cargo/config.toml' \ + ':crates/**' \ + ':src/**' \ + ':.github/workflows/ci.yaml' \ + ; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + env: + MERGE_BASE: ${{ steps.merge_base.outputs.sha }} + rust_tests: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} env: RUST_BACKTRACE: full - name: Run rust tests + # Named after the matrix entry rather than left to be named for it: a + # generated name lists every value in the entry, so adding or removing one + # renames the check and drops it from the required list. + name: Run rust tests (${{ matrix.os }}) runs-on: ${{ matrix.os }} timeout-minutes: 45 strategy: @@ -130,31 +90,70 @@ jobs: os: [macos-latest, ubuntu-latest, windows-2025] fail-fast: false steps: - - uses: actions/checkout@v6.0.2 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - components: clippy - - uses: Swatinem/rust-cache@v2 + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable - - name: Set up the Mac environment - run: brew install autoconf automake libtool - if: runner.os == 'macOS' + - name: Use a short Cargo home on Windows + if: runner.os == 'Windows' + shell: pwsh + run: | + "CARGO_HOME=$env:RUNNER_TEMP\c" >> $env:GITHUB_ENV + git config --global core.longpaths true - - name: run clippy - run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings + - name: Restore cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ${{ runner.temp }}/c/registry/index/ + ${{ runner.temp }}/c/registry/cache/ + ${{ runner.temp }}/c/git/db/ + target/ + key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}- + restore-keys: | + ${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-- + # Windows runners randomly crashes, https://github.com/actions/cache/issues/1754 + continue-on-error: true + + - name: Install macOS dependencies + uses: ./.github/actions/install-macos-deps - name: run rust tests - run: cargo test --workspace ${{ env.WORKSPACE_EXCLUDES }} --verbose --features threading ${{ env.CARGO_ARGS }} - if: runner.os != 'macOS' - - name: run rust tests - run: cargo test --workspace ${{ env.WORKSPACE_EXCLUDES }} --exclude rustpython-jit --verbose --features threading ${{ env.CARGO_ARGS }} - if: runner.os == 'macOS' + run: cargo test --locked --workspace --exclude rustpython-capi ${{ env.WORKSPACE_EXCLUDES }} --features threading ${{ env.CARGO_ARGS }} + env: + INSTA_WORKSPACE_ROOT: ${{ github.workspace }} + + - name: run c-api tests + working-directory: crates/capi + run: cargo test --locked + if: runner.os != 'Windows' # Requires pyo3 0.29+ on Windows + + - name: check compilation without host_env (sandbox mode) + run: | + cargo check --locked -p rustpython-vm --no-default-features --features compiler + cargo check --locked -p rustpython-stdlib --no-default-features --features compiler + cargo build --locked --no-default-features --features stdlib,importlib,stdio,encodings,freeze-stdlib + if: runner.os == 'Linux' - - name: check compilation without threading - run: cargo check ${{ env.CARGO_ARGS }} + - name: sandbox smoke test + run: | + target/debug/rustpython extra_tests/snippets/sandbox_smoke.py + target/debug/rustpython extra_tests/snippets/stdlib_re.py + if: runner.os == 'Linux' - name: Test openssl build - run: cargo build --no-default-features --features ssl-openssl + run: cargo build --locked --no-default-features --features ssl-openssl + if: runner.os == 'Linux' + + - name: Test vendored OpenSSL build + run: cargo build --locked --no-default-features --features ssl-openssl-vendor if: runner.os == 'Linux' # - name: Install tk-dev for tkinter build @@ -172,98 +171,114 @@ jobs: if: runner.os == 'Linux' - name: run update_lib tests - run: cargo run -- -m unittest discover -s scripts/update_lib/tests -v + run: cargo run --locked -- -m unittest discover -s scripts/update_lib/tests -v env: PYTHONPATH: scripts if: runner.os == 'Linux' - - name: prepare Intel MacOS build - uses: dtolnay/rust-toolchain@stable - with: + cargo_check: + # Named after the matrix entry rather than left to be named for it: a + # generated name lists every value in the entry, so adding or removing one + # renames the check and drops it from the required list. + name: cargo check (${{ matrix.os }}, ${{ matrix.target }}) + runs-on: ${{ matrix.os }} + needs: + - determine_changes + if: | + ( + !contains(github.event.pull_request.labels.*.name, 'skip:ci') && + needs.determine_changes.outputs.rust_code == 'true' + ) || github.ref == 'refs/heads/main' + strategy: + matrix: + include: + - os: ubuntu-latest + target: aarch64-linux-android + - os: ubuntu-latest + target: i686-unknown-linux-gnu + dependencies: + gcc-multilib: true + - os: ubuntu-latest + target: i686-unknown-linux-musl + dependencies: + musl-tools: true + skip_ssl: true + - os: ubuntu-latest + target: wasm32-wasip2 + skip_ssl: true + - os: ubuntu-latest + target: x86_64-unknown-freebsd + skip_ssl: true + - os: ubuntu-latest + target: aarch64-unknown-linux-gnu + dependencies: + gcc-aarch64-linux-gnu: true + - os: ubuntu-latest + target: x86_64-pc-windows-gnu + dependencies: + gcc-mingw-w64-x86-64: true + - os: macos-latest + target: aarch64-apple-ios + - os: macos-latest target: x86_64-apple-darwin - if: runner.os == 'macOS' - - name: Check compilation for Intel MacOS - run: cargo check --target x86_64-apple-darwin - if: runner.os == 'macOS' - - name: prepare iOS build - uses: dtolnay/rust-toolchain@stable + fail-fast: false + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - target: aarch64-apple-ios - if: runner.os == 'macOS' - - name: Check compilation for iOS - run: cargo check --target aarch64-apple-ios ${{ env.CARGO_ARGS_NO_SSL }} - if: runner.os == 'macOS' + persist-credentials: false - exotic_targets: - if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} - name: Ensure compilation on various targets - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v6.0.2 - - uses: dtolnay/rust-toolchain@stable + - name: Install dependencies + uses: ./.github/actions/install-linux-deps + # zizmor has an issue with dynamic `with` + # with: ${{ matrix.dependencies || fromJSON('{}') }} with: - target: i686-unknown-linux-gnu + gcc-multilib: ${{ matrix.dependencies.gcc-multilib || false }} + musl-tools: ${{ matrix.dependencies.musl-tools || false }} + gcc-aarch64-linux-gnu: ${{ matrix.dependencies.gcc-aarch64-linux-gnu || false }} + gcc-mingw-w64-x86-64: ${{ matrix.dependencies.gcc-mingw-w64-x86-64 || false }} - - name: Install gcc-multilib and musl-tools - run: sudo apt-get update && sudo apt-get install gcc-multilib musl-tools - - name: Check compilation for x86 32bit - run: cargo check --target i686-unknown-linux-gnu ${{ env.CARGO_ARGS_NO_SSL }} + - name: Restore cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + # key won't match, will rely on restore-keys + key: ${{ runner.os }}-${{ matrix.target }} + restore-keys: | + ${{ runner.os }}-stable-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-${{ matrix.target }}- + ${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-- - uses: dtolnay/rust-toolchain@stable with: - target: aarch64-linux-android + target: ${{ matrix.target }} - name: Setup Android NDK + if: ${{ matrix.target == 'aarch64-linux-android' }} id: setup-ndk - uses: nttld/setup-ndk@v1 + uses: nttld/setup-ndk@ed92fe6cadad69be94a966a7ee3271275e62f779 # v1.6.0 with: ndk-version: r27 add-to-path: true - - name: Check compilation for android - run: cargo check --target aarch64-linux-android ${{ env.CARGO_ARGS_NO_SSL }} + - name: Append env conf to cargo + if: ${{ matrix.target == 'aarch64-linux-android' }} env: - CC_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang - AR_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar - CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang - - - uses: dtolnay/rust-toolchain@stable - with: - target: aarch64-unknown-linux-gnu - - - name: Install gcc-aarch64-linux-gnu - run: sudo apt install gcc-aarch64-linux-gnu - - name: Check compilation for aarch64 linux gnu - run: cargo check --target aarch64-unknown-linux-gnu ${{ env.CARGO_ARGS_NO_SSL }} - - - uses: dtolnay/rust-toolchain@stable - with: - target: i686-unknown-linux-musl - - - name: Check compilation for musl - run: cargo check --target i686-unknown-linux-musl ${{ env.CARGO_ARGS_NO_SSL }} - - - uses: dtolnay/rust-toolchain@stable - with: - target: x86_64-unknown-freebsd - - - name: Check compilation for freebsd - run: cargo check --target x86_64-unknown-freebsd ${{ env.CARGO_ARGS_NO_SSL }} - - - uses: dtolnay/rust-toolchain@stable - with: - target: x86_64-unknown-freebsd - - - name: Check compilation for freeBSD - run: cargo check --target x86_64-unknown-freebsd ${{ env.CARGO_ARGS_NO_SSL }} + NDK_PATH: ${{ steps.setup-ndk.outputs.ndk-path }} + run: | + { + echo "[env]" + echo "CC_aarch64_linux_android = \"${NDK_PATH}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang\"" - - uses: dtolnay/rust-toolchain@stable - with: - target: wasm32-wasip2 + echo "AR_aarch64_linux_android = \"${NDK_PATH}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar\"" - - name: Check compilation for wasip2 - run: cargo check --target wasm32-wasip2 ${{ env.CARGO_ARGS_NO_SSL }} + echo "CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER = \"${NDK_PATH}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang\"" + } >> .cargo/config.toml # - name: Prepare repository for redox compilation # run: bash scripts/redox/uncomment-cargo.sh @@ -273,149 +288,172 @@ jobs: # command: check # args: --ignore-rust-version + - name: Check compilation with threading + run: cargo check --locked --target "${{ matrix.target }}" ${{ env.CARGO_ARGS_NO_SSL }} --features threading + + - name: Check compilation with ssl + if: ${{ !matrix.skip_ssl }} + run: cargo check --locked --target "${{ matrix.target }}" ${{ env.CARGO_ARGS }} + snippets_cpython: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} env: RUST_BACKTRACE: full - name: Run snippets and cpython tests + # Tests that can be flaky when running with multiple processes `-j 2`. We will use `-j 1` for these. + FLAKY_MP_TESTS: >- + test_class + test_concurrent_futures + test_eintr + test_multiprocessing_fork + test_multiprocessing_forkserver + test_multiprocessing_spawn + # Named after the matrix entry rather than left to be named for it: a + # generated name lists every value in the entry, so adding or removing one + # renames the check and drops it from the required list. + name: Run snippets and cpython tests (${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest, ubuntu-latest, windows-2025] + include: + - os: macos-latest + extra_test_args: + - '-u all' + - '--timeout 600' + - '--dont-add-python-opts' + env_polluting_tests: [] + skips: [] + timeout: 50 + - os: ubuntu-latest + extra_test_args: + - '-u all' + - '--timeout 600' + - '--dont-add-python-opts' + env_polluting_tests: [] + skips: [] + timeout: 60 + - os: windows-2025 + extra_test_args: + - '-u all' + - '--timeout 600' + - '--dont-add-python-opts' + env_polluting_tests: [] + skips: [] + timeout: 50 fail-fast: false steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - uses: actions/setup-python@v6.2.0 + + - name: Use a short Cargo home on Windows + if: runner.os == 'Windows' + shell: pwsh + run: | + "CARGO_HOME=$env:RUNNER_TEMP\c" >> $env:GITHUB_ENV + git config --global core.longpaths true + + - name: Restore cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Set up the Mac environment - run: brew install autoconf automake libtool openssl@3 - if: runner.os == 'macOS' - - name: build rustpython - run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }} - if: runner.os == 'macOS' - - name: build rustpython - run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }},jit - if: runner.os != 'macOS' - - uses: actions/setup-python@v6.2.0 + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ${{ runner.temp }}/c/registry/index/ + ${{ runner.temp }}/c/registry/cache/ + ${{ runner.temp }}/c/git/db/ + target/ + key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}- + restore-keys: | + ${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-- + # Windows runners randomly crashes, https://github.com/actions/cache/issues/1754 + continue-on-error: true + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + + - name: Install macOS dependencies + uses: ./.github/actions/install-macos-deps with: - python-version: ${{ env.PYTHON_VERSION }} + openssl: true + + # Keep features in sync with update-caches.yml CARGO_ARGS. + - name: build rustpython + run: cargo build --locked --release --verbose --features=threading,jit ${{ env.CARGO_ARGS }} + - name: run snippets run: python -m pip install -r requirements.txt && pytest -v working-directory: ./extra_tests - - if: runner.os == 'Linux' - name: run cpython platform-independent tests - env: - RUSTPYTHON_SKIP_ENV_POLLUTERS: true - run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed --timeout 600 -v ${{ env.PLATFORM_INDEPENDENT_TESTS }} - timeout-minutes: 45 + - name: Detect available cores + id: cores + shell: bash + run: | + cores=$(python -c 'print(__import__("os").process_cpu_count())') + echo "cores=${cores}" >> "$GITHUB_OUTPUT" - - if: runner.os == 'Linux' - name: run cpython platform-dependent tests (Linux) + - name: Run CPython tests + run: | + target/release/rustpython -u -m test --slow-ci -j ${{ steps.cores.outputs.cores }} ${{ join(matrix.extra_test_args, ' ') }} -x ${{ env.FLAKY_MP_TESTS }} ${{ join(matrix.skips, ' ') }} + timeout-minutes: ${{ matrix.timeout }} env: RUSTPYTHON_SKIP_ENV_POLLUTERS: true - run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed --timeout 600 -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} - timeout-minutes: 45 - - if: runner.os == 'macOS' - name: run cpython platform-dependent tests (MacOS) - env: - RUSTPYTHON_SKIP_ENV_POLLUTERS: true - run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} - timeout-minutes: 45 + - name: Run flaky MP CPython tests + run: | + for attempt in $(seq 1 5); do + echo "::group::Attempt ${attempt}" - - if: runner.os == 'Windows' - name: run cpython platform-dependent tests (windows partial - fixme) - env: - RUSTPYTHON_SKIP_ENV_POLLUTERS: true - run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.WINDOWS_SKIPS }} - timeout-minutes: 45 + set +e + target/release/rustpython -u -m test --slow-ci -j 1 ${{ join(matrix.extra_test_args, ' ') }} ${{ env.FLAKY_MP_TESTS }} + status=$? + set -e - - if: runner.os == 'Linux' - name: run cpython tests to check if env polluters have stopped polluting (Common/Linux) - shell: bash - run: | - for thing in ${{ env.ENV_POLLUTING_TESTS_COMMON }} ${{ env.ENV_POLLUTING_TESTS_LINUX }}; do - for i in $(seq 1 10); do - set +e - target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v ${thing} - exit_code=$? - set -e - if [ ${exit_code} -eq 3 ]; then - echo "Test ${thing} polluted the environment on attempt ${i}." - break - fi - done - if [ ${exit_code} -ne 3 ]; then - echo "Test ${thing} is no longer polluting the environment after ${i} attempts!" - echo "Please remove ${thing} from either ENV_POLLUTING_TESTS_COMMON or ENV_POLLUTING_TESTS_LINUX in '.github/workflows/ci.yaml'." - echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}." - if [ ${exit_code} -ne 0 ]; then - echo "Test ${thing} failed with exit code ${exit_code}." - echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip." - fi - exit 1 + echo "::endgroup::" + + if [ $status -eq 0 ]; then + exit 0 fi done - timeout-minutes: 15 - - if: runner.os == 'macOS' - name: run cpython tests to check if env polluters have stopped polluting (Common/macOS) + exit 1 + timeout-minutes: ${{ matrix.timeout }} shell: bash - run: | - for thing in ${{ env.ENV_POLLUTING_TESTS_COMMON }} ${{ env.ENV_POLLUTING_TESTS_MACOS }}; do - for i in $(seq 1 10); do - set +e - target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v ${thing} - exit_code=$? - set -e - if [ ${exit_code} -eq 3 ]; then - echo "Test ${thing} polluted the environment on attempt ${i}." - break - fi - done - if [ ${exit_code} -ne 3 ]; then - echo "Test ${thing} is no longer polluting the environment after ${i} attempts!" - echo "Please remove ${thing} from either ENV_POLLUTING_TESTS_COMMON or ENV_POLLUTING_TESTS_MACOS in '.github/workflows/ci.yaml'." - echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}." - if [ ${exit_code} -ne 0 ]; then - echo "Test ${thing} failed with exit code ${exit_code}." - echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip." - fi - exit 1 - fi - done - timeout-minutes: 15 + env: + RUSTPYTHON_SKIP_ENV_POLLUTERS: true - - if: runner.os == 'Windows' - name: run cpython tests to check if env polluters have stopped polluting (Common/windows) + - name: run cpython tests to check if env polluters have stopped polluting shell: bash run: | - for thing in ${{ env.ENV_POLLUTING_TESTS_COMMON }} ${{ env.ENV_POLLUTING_TESTS_WINDOWS }}; do + IFS=' ' read -r -a target_array <<< "$TARGETS" + + for thing in "${target_array[@]}"; do for i in $(seq 1 10); do set +e - target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v ${thing} + target/release/rustpython -u -m test --slow-ci -u all -j 1 --timeout 600 --dont-add-python-opts "${thing}" exit_code=$? set -e - if [ ${exit_code} -eq 3 ]; then + if [ "${exit_code}" -eq 3 ]; then echo "Test ${thing} polluted the environment on attempt ${i}." break fi done - if [ ${exit_code} -ne 3 ]; then + if [ "${exit_code}" -ne 3 ]; then echo "Test ${thing} is no longer polluting the environment after ${i} attempts!" - echo "Please remove ${thing} from either ENV_POLLUTING_TESTS_COMMON or ENV_POLLUTING_TESTS_WINDOWS in '.github/workflows/ci.yaml'." + echo "Please remove ${thing} from matrix.env_polluting_tests in '.github/workflows/ci.yaml'." echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}." - if [ ${exit_code} -ne 0 ]; then + if [ "${exit_code}" -ne 0 ]; then echo "Test ${thing} failed with exit code ${exit_code}." echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip." fi exit 1 fi done + env: + TARGETS: ${{ join(matrix.env_polluting_tests, ' ') }} timeout-minutes: 15 - if: runner.os != 'Windows' @@ -424,70 +462,194 @@ jobs: mkdir site-packages target/release/rustpython --install-pip ensurepip --user target/release/rustpython -m pip install six + - name: Check that ensurepip succeeds. run: | target/release/rustpython -m ensurepip target/release/rustpython -c "import pip" + + - if: runner.os == 'Windows' + name: Check pip HTTPS with the Windows trust store + run: >- + target/release/rustpython -m pip download + --disable-pip-version-check + --no-cache-dir + --no-deps + --only-binary=:all: + --dest "$env:RUNNER_TEMP\rustpython-pip-smoke" + six + - if: runner.os != 'Windows' name: Check if pip inside venv is functional run: | target/release/rustpython -m venv testvenv testvenv/bin/rustpython -m pip install wheel - - if: runner.os != 'macOS' - name: Check whats_left is not broken - shell: bash - run: python -I scripts/whats_left.py --no-default-features --features "$(sed -e 's/--[^ ]*//g' <<< "${{ env.CARGO_ARGS }}" | tr -d '[:space:]'),threading,jit" - - if: runner.os == 'macOS' # TODO fix jit on macOS - name: Check whats_left is not broken (macOS) + + - name: Check whats_left is not broken shell: bash - run: python -I scripts/whats_left.py --no-default-features --features "$(sed -e 's/--[^ ]*//g' <<< "${{ env.CARGO_ARGS }}" | tr -d '[:space:]'),threading" # no jit on macOS for now + run: python -I scripts/whats_left.py ${{ env.CARGO_ARGS }} --features jit + + clippy: + # Named after the matrix entry rather than left to be named for it: a + # generated name lists every value in the entry, so adding or removing one + # renames the check and drops it from the required list. + name: clippy (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + needs: + - determine_changes + permissions: + contents: read + if: | + needs.determine_changes.outputs.rust_code == 'true' || + github.ref == 'refs/heads/main' + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - ubuntu-latest + - windows-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - name: Use a short Cargo home on Windows + if: runner.os == 'Windows' + shell: pwsh + run: | + "CARGO_HOME=$env:RUNNER_TEMP\c" >> $env:GITHUB_ENV + git config --global core.longpaths true + + - name: Restore cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ${{ runner.temp }}/c/registry/index/ + ${{ runner.temp }}/c/registry/cache/ + ${{ runner.temp }}/c/git/db/ + target/ + key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}- + restore-keys: | + ${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-- + # Windows runners randomly crashes, https://github.com/actions/cache/issues/1754 + continue-on-error: true + + - name: Clippy + run: cargo clippy --locked --keep-going ${{ env.CARGO_ARGS }} --workspace --all-targets ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings + + cargo_shear: + name: cargo shear + runs-on: ubuntu-latest + needs: + - determine_changes + permissions: + contents: read + if: | + needs.determine_changes.outputs.rust_code == 'true' || + github.ref == 'refs/heads/main' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable + + - uses: cargo-bins/cargo-binstall@75b4bfae1b2c753a6806bbce6e6cb89b602de33c # v1.22.0 + + - name: cargo shear + run: | + cargo binstall --no-confirm cargo-shear + cargo shear --locked lint: - name: Check Rust code with clippy + name: Lint runs-on: ubuntu-latest + permissions: + contents: read + checks: write + issues: write + pull-requests: write + security-events: write # for zizmor steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: dtolnay/rust-toolchain@stable with: - components: clippy + components: rustfmt - - name: run clippy on wasm - run: cargo clippy --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings + - name: actionlint + uses: reviewdog/action-actionlint@dbe5299849118fd6f099ba563d263d770955a64a # v1.73.2 - - name: Ensure docs generate no warnings - run: cargo doc --locked + - name: zizmor + uses: zizmorcore/zizmor-action@70fb788f84895a7701f5643d103d587e460b5c99 # v0.6.3 + + - name: restore prek cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: prek-${{ hashFiles('.pre-commit-config.yaml') }} + path: ~/.cache/prek + + - name: install prek + id: prek + uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0 + with: + cache: false + show-verbose-logs: false + install-only: true + + - name: prek run + run: prek run --show-diff-on-failure --color=always --all-files - - name: Ensure Lib/_opcode_metadata is updated + - name: Get target CPython version + id: cpython-version run: | - python scripts/generate_opcode_metadata.py - if [ -n "$(git status --porcelain)" ]; then - exit 1 - fi + version=$(cat .python-version) + echo "version=${version}" >> "$GITHUB_OUTPUT" - - name: Install ruff - uses: astral-sh/ruff-action@57714a7c8a2e59f32539362ba31877a1957dded1 # v3.5.1 + - name: Clone CPython + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - version: "0.14.11" - args: "--version" + repository: python/cpython + path: cpython + ref: "v${{ steps.cpython-version.outputs.version }}" + persist-credentials: false - - run: ruff check --diff + - name: prek run (manual stage) + run: prek run --show-diff-on-failure --color=always --all-files --hook-stage manual + env: + CPYTHON_ROOT: ${{ github.workspace }}/cpython - - run: ruff format --check + - name: save prek cache + if: ${{ github.ref == 'refs/heads/main' }} # only save on main + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: prek-${{ hashFiles('.pre-commit-config.yaml') }} + path: ~/.cache/prek - - name: install prettier - run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH + - name: restore git permissions + if: ${{ !cancelled() }} + run: sudo chown -R "$(id -u):$(id -g)" .git - - name: check wasm code with prettier - # prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506 - run: cd wasm && git ls-files -z | xargs -0 prettier --check -u - # Keep cspell check as the last step. This is optional test. - - name: install extra dictionaries - run: npm install @cspell/dict-en_us @cspell/dict-cpp @cspell/dict-python @cspell/dict-rust @cspell/dict-win32 @cspell/dict-shell - - name: spell checker - uses: streetsidesoftware/cspell-action@v8 + - name: reviewdog + if: ${{ !cancelled() }} + uses: reviewdog/action-suggester@2558ba17e65a9039e73764a73009fc05fef28a46 # v1.24.3 with: - files: "**/*.rs" - incremental_files_only: true + level: warning + fail_level: error miri: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} @@ -497,21 +659,37 @@ jobs: env: NIGHTLY_CHANNEL: nightly steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.NIGHTLY_CHANNEL }} components: miri - - uses: Swatinem/rust-cache@v2 + - name: Restore cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }} + restore-keys: | + ${{ runner.os }}- - name: Run tests under miri - run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test -p rustpython-vm -- miri_test + run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test --locked -p rustpython-vm -- miri_test env: # miri-ignore-leaks because the type-object circular reference means that there will always be # a memory leak, at least until we have proper cyclic gc - MIRIFLAGS: "-Zmiri-ignore-leaks" + # miri-permissive-provenance because function pointer identity checks (slot comparisons) + # cast fn pointers to usize, which strips provenance — this is the standard pattern for + # fn pointer comparison in Rust and not a soundness issue + MIRIFLAGS: "-Zmiri-ignore-leaks -Zmiri-permissive-provenance" wasm: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} @@ -519,10 +697,32 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - name: Restore cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}- + restore-keys: | + ${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-- + ${{ runner.os }}- + + - name: cargo clippy + run: cargo --locked clippy --keep-going --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings - - uses: Swatinem/rust-cache@v2 - name: install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: install geckodriver @@ -530,25 +730,45 @@ jobs: wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz mkdir geckodriver tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver - - uses: actions/setup-python@v6.2.0 - with: - python-version: ${{ env.PYTHON_VERSION }} + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - run: python -m pip install -r requirements.txt working-directory: ./wasm/tests - - uses: actions/setup-node@v6 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + package-manager-cache: false + + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> "$GITHUB_OUTPUT" + + - name: Restore npm cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + # don't restore on main or release + if: github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' with: - cache: "npm" - cache-dependency-path: "wasm/demo/package-lock.json" + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: node-${{ runner.os }}-wasm-demo- + restore-keys: | + node-${{ runner.os }}-wasm-demo- + - name: run test run: | - export PATH=$PATH:`pwd`/../../geckodriver + driver_path="$(pwd)/../../geckodriver" + export PATH="$PATH:${driver_path}" npm install npm run test env: NODE_OPTIONS: "--openssl-legacy-provider" working-directory: ./wasm/demo - - uses: mwilliamson/setup-wabt-action@v3 - with: { wabt-version: "1.0.36" } + + - uses: mwilliamson/setup-wabt-action@427f2fdd70bc4dbc2e53c2eb4f19f66162d71bd2 # v4.0.0 + with: + wabt-version: "1.0.36" + - name: check wasm32-unknown without js run: | cd example_projects/wasm32_without_js/rustpython-without-js @@ -558,6 +778,7 @@ jobs: echo "ERROR: wasm32-unknown module expects imports from the host environment" >&2 fi cargo run --release --manifest-path wasm-runtime/Cargo.toml rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm + - name: build notebook demo if: github.ref == 'refs/heads/release' run: | @@ -567,34 +788,105 @@ jobs: env: NODE_OPTIONS: "--openssl-legacy-provider" working-directory: ./wasm/notebook + - name: Deploy demo to Github Pages if: success() && github.ref == 'refs/heads/release' - uses: peaceiris/actions-gh-pages@v4 + uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0 env: ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }} PUBLISH_DIR: ./wasm/demo/dist EXTERNAL_REPOSITORY: RustPython/demo PUBLISH_BRANCH: master + - name: Save npm cache + # Save only on main or release + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: node-${{ runner.os }}-wasm-demo-${{ hashFiles('wasm/demo/package-lock.json') }} + wasm-wasi: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Run snippets and cpython tests on wasm-wasi runs-on: ubuntu-latest timeout-minutes: 30 + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 - - uses: Swatinem/rust-cache@v2 + - name: Restore cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}- + restore-keys: | + ${{ runner.os }}-stable-wasm32-wasip1-${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-wasm32-wasip1- + ${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-- + - name: Setup Wasmer - uses: wasmerio/setup-wasmer@v3 + uses: wasmerio/setup-wasmer@24b15c95293d23f89c68bd40dac76338f773e924 # v3.1 + - name: Install clang - run: sudo apt-get update && sudo apt-get install clang -y + uses: ./.github/actions/install-linux-deps + with: + clang: true + - name: build rustpython - run: cargo build --release --target wasm32-wasip1 --features freeze-stdlib,stdlib --verbose + run: cargo build --locked --profile wasm-release --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib,stdio,importlib,host_env --verbose - name: run snippets - run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py + run: | + wasmer run --volume "$(pwd)" target/wasm32-wasip1/wasm-release/rustpython.wasm -- "$(pwd)/extra_tests/snippets/stdlib_random.py" + wasmer run --volume "$(pwd)" target/wasm32-wasip1/wasm-release/rustpython.wasm -- "$(pwd)/extra_tests/snippets/stdlib_time.py" - name: run cpython unittest - run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/Lib/test/test_int.py + run: wasmer run --volume "$(pwd)" target/wasm32-wasip1/wasm-release/rustpython.wasm -- "$(pwd)/Lib/test/test_int.py" + + cargo_doc: + needs: + - determine_changes + if: | + ( + !contains(github.event.pull_request.labels.*.name, 'skip:ci') && + needs.determine_changes.outputs.rust_code == 'true' + ) || github.ref == 'refs/heads/main' + env: + RUST_BACKTRACE: full + name: cargo doc + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable + + - name: Restore cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}- + restore-keys: | + ${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-- + + - name: cargo doc + run: cargo doc --locked diff --git a/.github/workflows/codspeed.yaml b/.github/workflows/codspeed.yaml new file mode 100644 index 00000000000..c7fc88352c1 --- /dev/null +++ b/.github/workflows/codspeed.yaml @@ -0,0 +1,114 @@ +on: + push: + branches: [main] + pull_request: + types: [unlabeled, opened, synchronize, reopened] + merge_group: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +name: CodSpeed + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +env: + CARGO_INCREMENTAL: 0 + CARGO_TERM_COLOR: always + CI: true + +jobs: + benchmarks: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} + name: Run benchmarks + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + id-token: write # for OpenID Connect authentication with CodSpeed + env: + CACHE_PATHS: | + # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci + ~/.cargo/.crates2.json + ~/.cargo/.crates.toml + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable + + # The benchmarks are linked against libpython through the pyo3 dev-dependency. + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + + - name: Restore the machine main last measured on + # A recording that cannot be fetched is read as "none yet", which the step + # below treats as nothing to disagree with. Failing here instead would + # turn a cache outage into a red benchmark job. + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: build/codspeed-environment-reference.json + key: codspeed-environment-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }} + restore-keys: codspeed-environment-${{ runner.os }}-${{ runner.arch }}- + + - name: Name the machine this run drew + id: environment + shell: bash + run: | + reference=build/codspeed-environment-reference.json + if [ "$GITHUB_REF" = refs/heads/main ]; then + args=(--record "$reference") + else + args=(--record build/codspeed-environment.json --reference "$reference") + fi + python3 scripts/codspeed-environment.py "${args[@]}" --github-output + + - name: Restore cache + if: ${{ steps.environment.outputs.match != 'false' }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.CACHE_PATHS }} + key: codspeed-v2-${{ hashFiles('**/Cargo.lock') }} + restore-keys: codspeed-v2- + continue-on-error: true + + - name: Install cargo-codspeed + if: ${{ steps.environment.outputs.match != 'false' }} + run: cargo install cargo-codspeed --locked --version ^5 + + - name: Build the benchmark targets + if: ${{ steps.environment.outputs.match != 'false' }} + run: cargo codspeed build --locked --measurement-mode simulation -p rustpython -p rustpython-sre_engine + + - name: Save cache + if: ${{ github.ref == 'refs/heads/main' }} # only save on main + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.CACHE_PATHS }} + key: codspeed-v2-${{ hashFiles('**/Cargo.lock') }} + continue-on-error: true + + - name: Run the benchmarks + if: ${{ steps.environment.outputs.match != 'false' }} + uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1 + with: + mode: simulation + run: cargo codspeed run -p rustpython -p rustpython-sre_engine + + - name: Record the machine this baseline was measured on + if: ${{ success() && github.ref == 'refs/heads/main' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: build/codspeed-environment-reference.json + key: codspeed-environment-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }} diff --git a/.github/workflows/comment-commands.yml b/.github/workflows/comment-commands.yml index 3f3402270ea..eb9e2f96d1a 100644 --- a/.github/workflows/comment-commands.yml +++ b/.github/workflows/comment-commands.yml @@ -18,4 +18,6 @@ jobs: steps: # Using REST API and not `gh issue edit`. https://github.com/cli/cli/issues/6235#issuecomment-1243487651 - run: | - curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees + curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ env.USER }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees + env: + USER: ${{ github.event.comment.user.login }} diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 0a546595a8c..2cf63a50a99 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -1,3 +1,5 @@ +name: Periodic checks/tasks + on: schedule: - cron: "0 0 * * 6" @@ -5,15 +7,14 @@ on: push: paths: - .github/workflows/cron-ci.yaml + branches: + - main pull_request: paths: - .github/workflows/cron-ci.yaml -name: Periodic checks/tasks - env: - CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl-rustls,jit - PYTHON_VERSION: "3.14.2" + CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls-aws-lc,jit,host_env jobs: # codecov collects code coverage data from the rust tests, python snippets and python test suite. @@ -23,29 +24,42 @@ jobs: runs-on: ubuntu-latest # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} + env: + INSTA_WORKSPACE_ROOT: ${{ github.workspace }} steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@cargo-llvm-cov - - uses: actions/setup-python@v6.2.0 + + - uses: taiki-e/install-action@1ed6d7be6168f6c9046541087ff549b6bc581fdf # v2.87.2 with: - python-version: ${{ env.PYTHON_VERSION }} + tool: cargo-llvm-cov + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - run: sudo apt-get update && sudo apt-get -y install lcov + - name: Run cargo-llvm-cov with Rust tests. - run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher --verbose --no-default-features --features stdlib,importlib,encodings,ssl-rustls,jit + run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher --exclude rustpython-capi --verbose --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls-aws-lc,jit,host_env + - name: Run cargo-llvm-cov with Python snippets. run: python scripts/cargo-llvm-cov.py continue-on-error: true + - name: Run cargo-llvm-cov with Python test suite. run: cargo llvm-cov --no-report run -- -m test -u all --slowest --fail-env-changed continue-on-error: true + - name: Prepare code coverage data run: cargo llvm-cov report --lcov --output-path='codecov.lcov' + - name: Upload to Codecov if: ${{ github.event_name != 'pull_request' }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: - file: ./codecov.lcov + files: ./codecov.lcov testdata: name: Collect regression test data @@ -53,19 +67,24 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: true + - uses: dtolnay/rust-toolchain@stable + - name: build rustpython run: cargo build --release --verbose + - name: collect tests data run: cargo run --release extra_tests/jsontests.py env: RUSTPYTHONPATH: ${{ github.workspace }}/Lib + - name: upload tests data to the website if: ${{ github.event_name != 'pull_request' }} env: SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }} - GITHUB_ACTOR: ${{ github.actor }} run: | echo "$SSHKEY" >~/github_key chmod 600 ~/github_key @@ -75,7 +94,7 @@ jobs: cd website cp ../extra_tests/cpython_tests_results.json ./_data/regrtests_results.json git add ./_data/regrtests_results.json - if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update regression test results" --author="$GITHUB_ACTOR"; then + if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update regression test results"; then git push fi @@ -85,24 +104,28 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.2 - - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v6.2.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - python-version: ${{ env.PYTHON_VERSION }} + persist-credentials: true + + - uses: dtolnay/rust-toolchain@stable + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - name: build rustpython run: cargo build --release --verbose + - name: Collect what is left data run: | chmod +x ./scripts/whats_left.py ./scripts/whats_left.py --features "ssl,sqlite" > whats_left.temp env: RUSTPYTHONPATH: ${{ github.workspace }}/Lib + - name: Upload data to the website if: ${{ github.event_name != 'pull_request' }} env: SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }} - GITHUB_ACTOR: ${{ github.actor }} run: | echo "$SSHKEY" >~/github_key chmod 600 ~/github_key @@ -133,7 +156,7 @@ jobs: } EOF git add -A - if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then + if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results"; then git push fi @@ -143,34 +166,44 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v6.0.2 - - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v6.2.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - python-version: ${{ env.PYTHON_VERSION }} + persist-credentials: true + + - uses: dtolnay/rust-toolchain@stable + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - run: cargo install cargo-criterion + - name: build benchmarks run: cargo build --release --benches + - name: collect execution benchmark data run: cargo criterion --bench execution + - name: collect microbenchmarks data run: cargo criterion --bench microbenchmarks + - name: restructure generated files run: | cd ./target/criterion/reports - find -type d -name cpython | xargs rm -rf - find -type d -name rustpython | xargs rm -rf - find -mindepth 2 -maxdepth 2 -name violin.svg | xargs rm -rf - find -type f -not -name violin.svg | xargs rm -rf - for file in $(find -type f -name violin.svg); do mv $file $(echo $file | sed -E "s_\./([^/]+)/([^/]+)/violin\.svg_./\1/\2.svg_"); done - find -mindepth 2 -maxdepth 2 -type d | xargs rm -rf + find . -type d -name cpython -print0 | xargs -0 rm -rf + find . -type d -name rustpython -print0 | xargs -0 rm -rf + find . -mindepth 2 -maxdepth 2 -name violin.svg -print0 | xargs -0 rm -rf + find . -type f -not -name violin.svg -print0 | xargs -0 rm -rf + find . -type f -name violin.svg -exec sh -c 'for file; do mv "$file" "$(echo "$file" | sed -E "s_\./([^/]+)/([^/]+)/violin\.svg_./\1/\2.svg_")"; done' _ {} + + find . -mindepth 2 -maxdepth 2 -type d -print0 | xargs -0 rm -rf cd .. mv reports/* . rmdir reports + - name: upload benchmark data to the website if: ${{ github.event_name != 'pull_request' }} env: SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }} + COMMIT_SHA: ${{ github.sha }} + REF_NAME: ${{ github.ref_name }} run: | echo "$SSHKEY" >~/github_key chmod 600 ~/github_key @@ -182,8 +215,8 @@ jobs: cp -r ../target/criterion ./assets/criterion printf '{\n "generated_at": "%s",\n "rustpython_commit": "%s",\n "rustpython_ref": "%s"\n}\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - "${{ github.sha }}" \ - "${{ github.ref_name }}" > ./_data/criterion-metadata.json + "$COMMIT_SHA" \ + "$REF_NAME" > ./_data/criterion-metadata.json git add ./assets/criterion ./_data/criterion-metadata.json if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update benchmark results"; then git push diff --git a/.github/workflows/lib-deps-check.yaml b/.github/workflows/lib-deps-check.yaml index a4b7128d830..6f147542ee6 100644 --- a/.github/workflows/lib-deps-check.yaml +++ b/.github/workflows/lib-deps-check.yaml @@ -4,10 +4,12 @@ on: pull_request_target: types: [opened, synchronize, reopened] paths: - - 'Lib/**' + - "Lib/**" + +permissions: {} concurrency: - group: lib-deps-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }} cancel-in-progress: true jobs: @@ -18,88 +20,113 @@ jobs: timeout-minutes: 10 steps: - name: Checkout base branch - uses: actions/checkout@v6.0.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Use base branch for scripts (security: don't run PR code with elevated permissions) ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 0 + persist-credentials: false - name: Fetch PR head + env: + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - git fetch origin ${{ github.event.pull_request.head.sha }} + git fetch origin "$PR_HEAD_SHA" - name: Checkout PR Lib files + env: + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | # Checkout only Lib/ directory from PR head for accurate comparison - git checkout ${{ github.event.pull_request.head.sha }} -- Lib/ + git checkout "$PR_HEAD_SHA" -- Lib/ - - name: Checkout CPython + - name: Get target CPython version + id: cpython-version run: | - git clone --depth 1 --branch v3.14.2 https://github.com/python/cpython.git cpython + version=$(cat .python-version) + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - name: Checkout CPython + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: python/cpython + path: cpython + ref: "v${{ steps.cpython-version.outputs.version }}" + fetch-depth: 1 + persist-credentials: false - name: Get changed Lib files - id: changed-files + id: all-changed-files + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | # Get the list of changed files under Lib/ - changed=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -- 'Lib/*.py' 'Lib/**/*.py' | head -50) - echo "Changed files:" - echo "$changed" - - # Extract unique module names - modules="" - for file in $changed; do - if [[ "$file" == Lib/test/* ]]; then - # Test files: Lib/test/test_pydoc.py -> test_pydoc, Lib/test/test_pydoc/foo.py -> test_pydoc - module=$(echo "$file" | sed -E 's|^Lib/test/||; s|\.py$||; s|/.*||') - # Skip non-test files in test/ (e.g., support.py, __init__.py) - if [[ ! "$module" == test_* ]]; then + { + echo 'changed<> "$GITHUB_OUTPUT" + + - name: Parse changed files + id: changed-files + run: | + from os import environ + + files = environ["FILES"] + modules = set() + for file in files.splitlines(): + file = file.strip() + + if file.startswith("Lib/test/"): + # Test files: + # Lib/test/test_pydoc.py -> test_pydoc + # Lib/test/test_pydoc/foo.py -> test_pydoc + module = file.removeprefix("Lib/test/").split("/")[0] + if not module.startswith("test_"): continue - fi - else - # Lib files: Lib/foo.py -> foo, Lib/foo/__init__.py -> foo - module=$(echo "$file" | sed -E 's|^Lib/||; s|/__init__\.py$||; s|\.py$||; s|/.*||') - fi - if [[ -n "$module" && ! " $modules " =~ " $module " ]]; then - modules="$modules $module" - fi - done - - modules=$(echo "$modules" | xargs) # trim whitespace - echo "Detected modules: $modules" - echo "modules=$modules" >> $GITHUB_OUTPUT + else: + # Lib files: + # Lib/foo.py -> foo + # Lib/foo/__init__.py -> foo + module = file.removeprefix("Lib/").split("/")[0] + + module = module.split(".")[0] + modules.add(module) + + print(f"{modules=}") + output = " ".join(sorted(modules)) + output_file = environ["GITHUB_OUTPUT"] + with open(output_file, mode="a", encoding="utf-8") as fd: + fd.write(f"modules={output}\n") + env: + FILES: ${{ steps.all-changed-files.outputs.changed }} + shell: python - name: Setup Python if: steps.changed-files.outputs.modules != '' - uses: actions/setup-python@v6.2.0 - with: - python-version: "3.12" + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - name: Run deps check if: steps.changed-files.outputs.modules != '' id: deps-check run: | # Run deps for all modules at once - python scripts/update_lib deps ${{ steps.changed-files.outputs.modules }} --depth 2 > /tmp/deps_output.txt 2>&1 || true - - # Read output for GitHub Actions - echo "deps_output<> $GITHUB_OUTPUT - cat /tmp/deps_output.txt >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - # Check if there's any meaningful output - if [ -s /tmp/deps_output.txt ]; then - echo "has_output=true" >> $GITHUB_OUTPUT - else - echo "has_output=false" >> $GITHUB_OUTPUT - fi + echo "deps_output<> "$GITHUB_OUTPUT" + output=$(python scripts/update_lib deps "${MODULES}" --depth 2 2>&1 || true) + echo "$output" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + env: + MODULES: ${{ steps.changed-files.outputs.modules }} - name: Post comment - if: steps.deps-check.outputs.has_output == 'true' - uses: marocchino/sticky-pull-request-comment@v2 + if: steps.deps-check.outputs.deps_output != '' + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: header: lib-deps-check number: ${{ github.event.pull_request.number }} - recreate: true message: | ## 📦 Library Dependencies @@ -113,7 +140,7 @@ jobs: - name: Remove comment if no Lib changes if: steps.changed-files.outputs.modules == '' - uses: marocchino/sticky-pull-request-comment@v2 + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 with: header: lib-deps-check number: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/oscca-pr.yml b/.github/workflows/oscca-pr.yml new file mode 100644 index 00000000000..38b67625ce0 --- /dev/null +++ b/.github/workflows/oscca-pr.yml @@ -0,0 +1,70 @@ +name: Manage OSCCA pull requests + +on: + pull_request_target: + types: [opened] + +permissions: {} + +jobs: + label-and-assign: + name: Label and assign OSCCA pull request + runs-on: ubuntu-slim + timeout-minutes: 5 + permissions: + pull-requests: write + steps: + - name: Label and assign pull request + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const osccaUsers = new Set( + [ + "2jiyong", + "chestnut1717", + "devyubin", + "fregataa", + "hyoinandout", + "HyoJongPark", + "JaceJung-dev", + "jinmay", + "jiwahn", + "kangdora", + "kim-jaedeok", + "kyokuping", + "leehanjeong", + "lms0806", + "lsahn-gh", + "moreal", + "name-of-okja", + "rlaisqls", + "seungje0612", + "shAn-kor", + "sigmaith", + "teddygood", + "widehyo1", + "YangSiJun528", + "zzarbttoo", + ].map((login) => login.toLowerCase()), + ); + const pullRequest = context.payload.pull_request; + const author = pullRequest.user.login; + + if (!osccaUsers.has(author.toLowerCase())) { + core.info(`${author} is not an OSCCA participant; skipping.`); + return; + } + + const issue = { + ...context.repo, + issue_number: pullRequest.number, + }; + + await github.rest.issues.addLabels({ + ...issue, + labels: ["z-ca-2026"], + }); + await github.rest.issues.addAssignees({ + ...issue, + assignees: [author], + }); diff --git a/.github/workflows/pr-auto-commit.yaml b/.github/workflows/pr-auto-commit.yaml deleted file mode 100644 index f34dd724e4c..00000000000 --- a/.github/workflows/pr-auto-commit.yaml +++ /dev/null @@ -1,120 +0,0 @@ -name: Auto-format PR - -# This workflow triggers when a PR is opened/updated -on: - pull_request_target: - types: [opened, synchronize, reopened] - branches: - - main - - release - -concurrency: - group: auto-format-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - auto_format: - permissions: - contents: write - pull-requests: write - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout PR branch - uses: actions/checkout@v6.0.2 - with: - ref: ${{ github.event.pull_request.head.sha }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - token: ${{ secrets.AUTO_COMMIT_PAT }} - fetch-depth: 0 - - - name: Setup Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - - name: Configure git - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - echo "" > /tmp/committed_commands.txt - - - name: Run cargo fmt - run: | - echo "Running cargo fmt --all on PR #${{ github.event.pull_request.number }}" - cargo fmt --all - if [ -n "$(git status --porcelain)" ]; then - git add -u - git commit -m "Auto-format: cargo fmt --all" - echo "- \`cargo fmt --all\`" >> /tmp/committed_commands.txt - fi - - - name: Install ruff - uses: astral-sh/ruff-action@57714a7c8a2e59f32539362ba31877a1957dded1 # v3.5.1 - with: - version: "0.14.11" - args: "--version" - - - name: Run ruff format - run: | - ruff format - if [ -n "$(git status --porcelain)" ]; then - git add -u - git commit -m "Auto-format: ruff format" - echo "- \`ruff format\`" >> /tmp/committed_commands.txt - fi - - - name: Run ruff check import sorting - run: | - ruff check --select I --fix - if [ -n "$(git status --porcelain)" ]; then - git add -u - git commit -m "Auto-format: ruff check --select I --fix" - echo "- \`ruff check --select I --fix\`" >> /tmp/committed_commands.txt - fi - - - name: Run generate_opcode_metadata.py - run: | - python scripts/generate_opcode_metadata.py - if [ -n "$(git status --porcelain)" ]; then - git add -u - git commit -m "Auto-generate: generate_opcode_metadata.py" - echo "- \`python scripts/generate_opcode_metadata.py\`" >> /tmp/committed_commands.txt - fi - - - name: Check for changes - id: check-changes - run: | - if [ "$(git rev-parse HEAD)" != "${{ github.event.pull_request.head.sha }}" ]; then - echo "has_changes=true" >> $GITHUB_OUTPUT - else - echo "has_changes=false" >> $GITHUB_OUTPUT - fi - - - name: Push formatting changes - if: steps.check-changes.outputs.has_changes == 'true' - run: | - git push origin HEAD:${{ github.event.pull_request.head.ref }} - - - name: Read committed commands - id: committed-commands - if: steps.check-changes.outputs.has_changes == 'true' - run: | - echo "list<> $GITHUB_OUTPUT - cat /tmp/committed_commands.txt >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Comment on PR - if: steps.check-changes.outputs.has_changes == 'true' - uses: marocchino/sticky-pull-request-comment@v2 - with: - number: ${{ github.event.pull_request.number }} - message: | - **Code has been automatically formatted** - - The code in this PR has been formatted using: - ${{ steps.committed-commands.outputs.list }} - Please pull the latest changes before pushing again: - ```bash - git pull origin ${{ github.event.pull_request.head.ref }} - ``` diff --git a/.github/workflows/pyperformance-comment.yaml b/.github/workflows/pyperformance-comment.yaml new file mode 100644 index 00000000000..565d933e967 --- /dev/null +++ b/.github/workflows/pyperformance-comment.yaml @@ -0,0 +1,81 @@ +on: + workflow_run: + workflows: [pyperformance] + types: [completed] + +name: pyperformance comment + +# Same split as `codspeed-comment.yaml`: a `pull_request` run from a fork gets +# a read-only token however it is configured, so the comparison is computed +# there and handed over as an artifact, and this workflow -- which +# `workflow_run` always runs in the base repo's context -- is the one trusted +# with a token that may write the comment. +permissions: + contents: read + actions: read # needed by download-artifact to fetch another run's artifact + pull-requests: write + +jobs: + comment: + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + steps: + - name: Download the comparison produced by the pyperformance workflow + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: pyperformance-diff + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + continue-on-error: true # absent when the job was skipped or dispatched + + - name: Post or update the PR comment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const fs = require('fs'); + if (!fs.existsSync('diff.md')) { + core.info('No diff.md in this run; nothing to comment.'); + return; + } + + const marker = ''; + const run = context.payload.workflow_run; + let prNumber = (run.pull_requests || []).map((pr) => pr.number)[0]; + if (!prNumber) { + const { data } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: run.head_sha, + }); + prNumber = data.find((pr) => pr.state === 'open')?.number; + } + if (!prNumber) { + core.info(`No open pull request for ${run.head_sha}; nothing to comment.`); + return; + } + + const raw = fs.readFileSync('diff.md', 'utf8').slice(0, 60000); + const body = `${marker}\n\n${raw.replace(/'; + + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + }); + const existing = comments.find((c) => c.body && c.body.includes(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body, + }); + } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9a6d0ad9838..6fa03af2373 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,83 +12,96 @@ on: required: false default: true -permissions: - contents: write - env: - CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include +permissions: {} + jobs: build: - runs-on: ${{ matrix.platform.runner }} + runs-on: ${{ matrix.os }} # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} + permissions: + contents: read strategy: matrix: - platform: - - runner: ubuntu-latest + include: + - os: ubuntu-latest target: x86_64-unknown-linux-gnu -# - runner: ubuntu-latest -# target: i686-unknown-linux-gnu -# - runner: ubuntu-latest -# target: aarch64-unknown-linux-gnu -# - runner: ubuntu-latest -# target: armv7-unknown-linux-gnueabi -# - runner: ubuntu-latest -# target: s390x-unknown-linux-gnu -# - runner: ubuntu-latest -# target: powerpc64le-unknown-linux-gnu - - runner: macos-latest + - os: macos-latest target: aarch64-apple-darwin -# - runner: macos-latest -# target: x86_64-apple-darwin - - runner: windows-2025 + - os: windows-2025 target: x86_64-pc-windows-msvc -# - runner: windows-2025 -# target: i686-pc-windows-msvc -# - runner: windows-2025 -# target: aarch64-pc-windows-msvc + # - os: ubuntu-latest + # target: i686-unknown-linux-gnu + # - os: ubuntu-latest + # target: aarch64-unknown-linux-gnu + # - os: ubuntu-latest + # target: armv7-unknown-linux-gnueabi + # - os: ubuntu-latest + # target: s390x-unknown-linux-gnu + # - os: ubuntu-latest + # target: powerpc64le-unknown-linux-gnu + # - os: macos-latest + # target: x86_64-apple-darwin + # - os: windows-2025 + # target: i686-pc-windows-msvc + # - os: windows-2025 + # target: aarch64-pc-windows-msvc fail-fast: false steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable - - uses: cargo-bins/cargo-binstall@main + with: + target: ${{ matrix.target }} - - name: Set up Environment - shell: bash - run: rustup target add ${{ matrix.platform.target }} - - name: Set up MacOS Environment - run: brew install autoconf automake libtool - if: runner.os == 'macOS' + - name: Use a short Cargo home on Windows + if: runner.os == 'Windows' + shell: pwsh + run: | + "CARGO_HOME=$env:RUNNER_TEMP\c" >> $env:GITHUB_ENV + git config --global core.longpaths true + + - name: Install macOS dependencies + uses: ./.github/actions/install-macos-deps + with: + autoconf: true + automake: true + libtool: true - name: Build RustPython - run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }} - if: runner.os == 'macOS' - - name: Build RustPython - run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }},jit - if: runner.os != 'macOS' + run: cargo build --release --target=${{ matrix.target }} --verbose --no-default-features --features stdlib,stdio,importlib,encodings,sqlite,host_env,ssl-rustls-aws-lc,threading,jit - name: Rename Binary - run: cp target/${{ matrix.platform.target }}/release/rustpython target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }} + run: cp target/${{ matrix.target }}/release/rustpython target/rustpython-release-${{ runner.os }}-${{ matrix.target }} if: runner.os != 'Windows' + - name: Rename Binary - run: cp target/${{ matrix.platform.target }}/release/rustpython.exe target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}.exe + run: cp target/${{ matrix.target }}/release/rustpython.exe target/rustpython-release-${{ runner.os }}-${{ matrix.target }}.exe if: runner.os == 'Windows' - name: Upload Binary Artifacts - uses: actions/upload-artifact@v6.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: rustpython-release-${{ runner.os }}-${{ matrix.platform.target }} - path: target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}* + name: rustpython-release-${{ runner.os }}-${{ matrix.target }} + path: target/rustpython-release-${{ runner.os }}-${{ matrix.target }}* build-wasm: runs-on: ubuntu-latest # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} + permissions: + contents: read steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable with: targets: wasm32-wasip1 @@ -100,16 +113,22 @@ jobs: run: cp target/wasm32-wasip1/release/rustpython.wasm target/rustpython-release-wasm32-wasip1.wasm - name: Upload Binary Artifacts - uses: actions/upload-artifact@v6.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: rustpython-release-wasm32-wasip1 path: target/rustpython-release-wasm32-wasip1.wasm - name: install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - - uses: actions/setup-node@v6 - - uses: mwilliamson/setup-wabt-action@v3 - with: { wabt-version: "1.0.30" } + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + package-manager-cache: false + + - uses: mwilliamson/setup-wabt-action@427f2fdd70bc4dbc2e53c2eb4f19f66162d71bd2 # v4.0.0 + with: + wabt-version: "1.0.30" + - name: build demo run: | npm install @@ -117,6 +136,7 @@ jobs: env: NODE_OPTIONS: "--openssl-legacy-provider" working-directory: ./wasm/demo + - name: build notebook demo run: | npm install @@ -125,8 +145,10 @@ jobs: env: NODE_OPTIONS: "--openssl-legacy-provider" working-directory: ./wasm/notebook + - name: Deploy demo to Github Pages - uses: peaceiris/actions-gh-pages@v4 + if: ${{ github.repository == 'RustPython/RustPython' }} + uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0 with: deploy_key: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }} publish_dir: ./wasm/demo/dist @@ -138,31 +160,31 @@ jobs: # Disable this scheduled job when running on a fork. if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} needs: [build, build-wasm] + permissions: + contents: write # for creating a release steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Download Binary Artifacts - uses: actions/download-artifact@v7.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: bin pattern: rustpython-* merge-multiple: true - name: Create Lib Archive - run: | - zip -r bin/rustpython-lib.zip Lib/ + run: zip -r bin/rustpython-lib.zip Lib/ - name: List Binaries run: | ls -lah bin/ file bin/* + - name: Create Release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ github.ref_name }} - run: ${{ github.run_number }} run: | - if [[ "${{ github.event.inputs.pre-release }}" == "false" ]]; then + if [[ "${PRE_RELEASE_INPUT}" == "false" ]]; then RELEASE_TYPE_NAME=Release PRERELEASE_ARG= else @@ -179,3 +201,8 @@ jobs: --generate-notes \ $PRERELEASE_ARG \ bin/rustpython-release-* + env: + GH_TOKEN: ${{ github.token }} + tag: ${{ github.ref_name }} + run: ${{ github.run_number }} + PRE_RELEASE_INPUT: ${{ github.event.inputs.pre-release }} diff --git a/.github/workflows/update-caches.yml b/.github/workflows/update-caches.yml new file mode 100644 index 00000000000..59ff6006fc8 --- /dev/null +++ b/.github/workflows/update-caches.yml @@ -0,0 +1,88 @@ +name: Update Actions Caches + +permissions: + contents: read + +on: + workflow_dispatch: + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +env: + CARGO_INCREMENTAL: 0 + CARGO_TERM_COLOR: always + CARGO_PROFILE_TEST_DEBUG: 0 + CARGO_PROFILE_DEV_DEBUG: 0 + CARGO_PROFILE_RELEASE_DEBUG: 0 + # Keep feature list in sync with CI's release build in .github/workflows/ci.yaml. + CARGO_ARGS: --workspace --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls-aws-lc,host_env,threading,jit --exclude rustpython_wasm --exclude rustpython-venvlauncher + +jobs: + build-caches: + name: Build Caches + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: macos-latest + toolchain: stable + target: "" + - os: ubuntu-latest + toolchain: stable + target: "" + - os: windows-latest + toolchain: stable + target: "" + steps: + - name: Checkout RustPython main branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: RustPython/RustPython + ref: main + persist-credentials: false + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ matrix.toolchain }} + target: ${{ matrix.target }} + + - name: Use a short Cargo home on Windows + if: runner.os == 'Windows' + shell: pwsh + run: | + "CARGO_HOME=$env:RUNNER_TEMP\c" >> $env:GITHUB_ENV + git config --global core.longpaths true + + - name: Install macos dependencies + uses: ./.github/actions/install-macos-deps + with: + openssl: true + + - name: Build dev cache # dev profile used by check & doc + run: cargo build --locked --profile dev ${{ env.CARGO_ARGS }} + + - name: Build test cache + run: cargo build --locked --profile test ${{ env.CARGO_ARGS }} + + - name: Build release cache + run: cargo build --locked --profile release ${{ env.CARGO_ARGS }} + + - name: Save cache + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ${{ runner.temp }}/c/registry/index/ + ${{ runner.temp }}/c/registry/cache/ + ${{ runner.temp }}/c/git/db/ + target/ + key: ${{ runner.os }}-${{ matrix.toolchain }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('Cargo.lock') }}-${{ github.sha }} diff --git a/.github/workflows/update-doc-db.yml b/.github/workflows/update-doc-db.yml index 1fd3b930985..c42235c86df 100644 --- a/.github/workflows/update-doc-db.yml +++ b/.github/workflows/update-doc-db.yml @@ -1,7 +1,6 @@ name: Update doc DB -permissions: - contents: write +permissions: {} on: workflow_dispatch: @@ -9,11 +8,11 @@ on: python-version: description: Target python version to generate doc db for type: string - default: "3.14.2" - ref: - description: Branch to commit to (leave empty for current branch) + default: "3.14.7" + base-ref: + description: Base branch to create the update branch from type: string - default: "" + default: "main" defaults: run: @@ -21,6 +20,8 @@ defaults: jobs: generate: + permissions: + contents: read runs-on: ${{ matrix.os }} strategy: matrix: @@ -29,20 +30,20 @@ jobs: - windows-latest - macos-latest steps: - - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false sparse-checkout: | crates/doc - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ inputs.python-version }} - name: Generate docs run: python crates/doc/generate.py - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: doc-db-${{ inputs.python-version }}-${{ matrix.os }} path: "crates/doc/generated/*.json" @@ -53,20 +54,30 @@ jobs: merge: runs-on: ubuntu-latest needs: generate + permissions: + contents: write + pull-requests: write steps: - - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ inputs.ref || github.ref }} - token: ${{ secrets.AUTO_COMMIT_PAT }} + persist-credentials: true + ref: ${{ inputs.base-ref }} + + - name: Create update branch + run: git switch -c "update-doc-${PYTHON_VERSION}" + env: + PYTHON_VERSION: ${{ inputs.python-version }} - name: Download generated doc DBs - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: "doc-db-${{ inputs.python-version }}-**" path: crates/doc/generated/ merge-multiple: true - name: Transform JSON + env: + PYTHON_VERSION: ${{ inputs.python-version }} run: | # Merge all artifacts jq -s "add" --sort-keys crates/doc/generated/*.json > crates/doc/generated/merged.json @@ -76,19 +87,18 @@ jobs: OUTPUT_FILE='crates/doc/src/data.inc.rs' - echo -n '' > $OUTPUT_FILE - - echo '// This file was auto-generated by `.github/workflows/update-doc-db.yml`.' >> $OUTPUT_FILE - echo "// CPython version: ${{ inputs.python-version }}" >> $OUTPUT_FILE - echo '// spell-checker: disable' >> $OUTPUT_FILE - - echo '' >> $OUTPUT_FILE - - echo "pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! {" >> $OUTPUT_FILE - cat crates/doc/generated/raw_entries.txt >> $OUTPUT_FILE - echo '};' >> $OUTPUT_FILE - - - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + # shellcheck disable=SC2016 + { + echo '// This file was auto-generated by `.github/workflows/update-doc-db.yml`.' + echo "// CPython version: ${PYTHON_VERSION}" + echo '// spell-checker: disable' + echo '' + echo "pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! {" + cat crates/doc/generated/raw_entries.txt + echo '};' + } > "$OUTPUT_FILE" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: doc-db-${{ inputs.python-version }} path: "crates/doc/src/data.inc.rs" @@ -96,13 +106,20 @@ jobs: retention-days: 7 overwrite: true - - name: Commit and push (non-main branches only) - if: github.ref != 'refs/heads/main' && inputs.ref != 'main' + - name: Commit, push and create PR + env: + GH_TOKEN: ${{ github.token }} + PYTHON_VERSION: ${{ inputs.python-version }} + BASE_REF: ${{ inputs.base-ref }} run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" if [ -n "$(git status --porcelain)" ]; then git add crates/doc/src/data.inc.rs - git commit -m "Update doc DB for CPython ${{ inputs.python-version }}" - git push + git commit -m "Update doc DB for CPython ${PYTHON_VERSION}" + git push -u origin HEAD + gh pr create \ + --base "${BASE_REF}" \ + --title "Update doc DB for CPython ${PYTHON_VERSION}" \ + --body "Auto-generated by update-doc-db workflow." fi diff --git a/.github/workflows/update-libs-status.yaml b/.github/workflows/update-libs-status.yaml new file mode 100644 index 00000000000..c1c656d2068 --- /dev/null +++ b/.github/workflows/update-libs-status.yaml @@ -0,0 +1,96 @@ +name: Updated libs status + +on: + push: + branches: + - main + paths: + - "Lib/**" + workflow_dispatch: + +permissions: + contents: read + issues: write + +env: + ISSUE_ID: "6839" + +jobs: + update-issue: + runs-on: ubuntu-latest + if: ${{ github.repository == 'RustPython/RustPython' }} + steps: + - name: Clone RustPython + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: rustpython + persist-credentials: false + sparse-checkout: |- + Lib + scripts/update_lib + .python-version + + - name: Get target CPython version + id: cpython-version + run: | + version=$(cat rustpython/.python-version) + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - name: Clone CPython + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: python/cpython + path: cpython + ref: "v${{ steps.cpython-version.outputs.version }}" + persist-credentials: false + sparse-checkout: | + Lib + + - name: Get current date + id: current_date + run: | + now=$(date -u +"%Y-%m-%d %H:%M:%S") + echo "date=$now" >> "$GITHUB_OUTPUT" + + - name: Write body prefix + run: | + cat > body.txt < + + ## Summary + + Check \`scripts/update_lib\` for tools. As a note, the current latest Python version is \`${{ steps.cpython-version.outputs.version }}\`. + + Previous versions' issues as reference + - 3.13: #5529 + + + + ## Details + + ${{ steps.current_date.outputs.date }} (UTC) + \`\`\`shell + $ python3 scripts/update_lib todo --done + \`\`\` + EOF + + - name: Run todo + run: python3 rustpython/scripts/update_lib todo --cpython cpython --lib rustpython/Lib --done >> body.txt + + - name: Update GH issue + run: gh issue edit ${{ env.ISSUE_ID }} --body-file ../body.txt + env: + GH_TOKEN: ${{ github.token }} + working-directory: rustpython + + diff --git a/.github/workflows/upgrade-pylib.lock.yml b/.github/workflows/upgrade-pylib.lock.yml new file mode 100644 index 00000000000..36d4b111281 --- /dev/null +++ b/.github/workflows/upgrade-pylib.lock.yml @@ -0,0 +1,1095 @@ +# +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ +# | _ |/ _` |/ _ \ '_ \| __| |/ __| +# | | | | (_| | __/ | | | |_| | (__ +# \_| |_/\__, |\___|_| |_|\__|_|\___| +# __/ | +# _ _ |___/ +# | | | | / _| | +# | | | | ___ _ __ _ __| |_| | _____ ____ +# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| +# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ +# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ +# +# This file was automatically generated by gh-aw (v0.43.22). DO NOT EDIT. +# +# To update this file, edit the corresponding .md file and run: +# gh aw compile +# Not all edits will cause changes to this file. +# +# For more information: https://github.github.com/gh-aw/introduction/overview/ +# +# Pick an out-of-sync Python library from the todo list and upgrade it +# by running `scripts/update_lib quick`, then open a pull request. +# +# frontmatter-hash: 3129480d6628afe028911bb8b31b6bb3b5eb251e395f00ed0677922cd21727cb + +name: "Upgrade Python Library" +"on": + workflow_dispatch: + inputs: + name: + description: Module name to upgrade (leave empty to auto-pick) + required: false + type: string + +permissions: {} + +concurrency: + group: "gh-aw-${{ github.workflow }}" + +run-name: "Upgrade Python Library" + +env: + ISSUE_ID: "6839" + PYTHON_VERSION: v3.14.3 + +# Cache configuration from frontmatter was processed and added to the main job steps + +jobs: + activation: + runs-on: ubuntu-slim + permissions: + contents: read + outputs: + comment_id: "" + comment_repo: "" + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@ff62cdbec36230acbae869ddb28806e8eca01ea1 # v0.87.10 + with: + destination: /opt/gh-aw/actions + - name: Check workflow file timestamps + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_WORKFLOW_FILE: "upgrade-pylib.lock.yml" + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + + agent: + needs: activation + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_AW_ASSETS_ALLOWED_EXTS: "" + GH_AW_ASSETS_BRANCH: "" + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs.jsonl + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + outputs: + checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + has_patch: ${{ steps.collect_output.outputs.has_patch }} + model: ${{ steps.generate_aw_info.outputs.model }} + output: ${{ steps.collect_output.outputs.output }} + output_types: ${{ steps.collect_output.outputs.output_types }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@ff62cdbec36230acbae869ddb28806e8eca01ea1 # v0.87.10 + with: + destination: /opt/gh-aw/actions + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + - name: Create gh-aw temp directory + run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh + # Cache configuration from frontmatter processed below + - name: Cache (cpython-lib-${{ env.PYTHON_VERSION }}) + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: cpython-lib-${{ env.PYTHON_VERSION }} + path: cpython + restore-keys: | + cpython-lib- + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Checkout PR branch + id: checkout-pr + if: | + github.event.pull_request + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/checkout_pr_branch.cjs'); + await main(); + - name: Generate agentic run info + id: generate_aw_info + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const fs = require('fs'); + + const awInfo = { + engine_id: "copilot", + engine_name: "GitHub Copilot CLI", + model: process.env.GH_AW_MODEL_AGENT_COPILOT || "", + version: "", + agent_version: "0.0.409", + cli_version: "v0.43.22", + workflow_name: "Upgrade Python Library", + experimental: false, + supports_tools_allowlist: true, + supports_http_transport: true, + run_id: context.runId, + run_number: context.runNumber, + run_attempt: process.env.GITHUB_RUN_ATTEMPT, + repository: context.repo.owner + '/' + context.repo.repo, + ref: context.ref, + sha: context.sha, + actor: context.actor, + event_name: context.eventName, + staged: false, + allowed_domains: ["defaults","rust","python"], + firewall_enabled: true, + awf_version: "v0.16.4", + awmg_version: "", + steps: { + firewall: "squid" + }, + created_at: new Date().toISOString() + }; + + // Write to /tmp/gh-aw directory to avoid inclusion in PR + const tmpPath = '/tmp/gh-aw/aw_info.json'; + fs.writeFileSync(tmpPath, JSON.stringify(awInfo, null, 2)); + console.log('Generated aw_info.json at:', tmpPath); + console.log(JSON.stringify(awInfo, null, 2)); + + // Set model as output for reuse in other steps/jobs + core.setOutput('model', awInfo.model); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Install GitHub Copilot CLI + run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.409 + - name: Install awf binary + run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.16.4 + - name: Determine automatic lockdown mode for GitHub MCP server + id: determine-automatic-lockdown + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + with: + script: | + const determineAutomaticLockdown = require('/opt/gh-aw/actions/determine_automatic_lockdown.cjs'); + await determineAutomaticLockdown(github, context, core); + - name: Download container images + run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.16.4 ghcr.io/github/gh-aw-firewall/squid:0.16.4 ghcr.io/github/gh-aw-mcpg:v0.1.4 ghcr.io/github/github-mcp-server:v0.30.3 node:lts-alpine + - name: Write Safe Outputs Config + run: | + mkdir -p /opt/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' + {"create_pull_request":{"expires":30},"missing_data":{},"missing_tool":{},"noop":{"max":1}} + GH_AW_SAFE_OUTPUTS_CONFIG_EOF + cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' + [ + { + "description": "Create a new GitHub pull request to propose code changes. Use this after making file edits to submit them for review and merging. The PR will be created from the current branch with your committed changes. For code review comments on an existing PR, use create_pull_request_review_comment instead. CONSTRAINTS: Maximum 1 pull request(s) can be created. Title will be prefixed with \"Update \". Labels [pylib-sync] will be automatically added.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "body": { + "description": "Detailed PR description in Markdown. Include what changes were made, why, testing notes, and any breaking changes. Do NOT repeat the title as a heading.", + "type": "string" + }, + "branch": { + "description": "Source branch name containing the changes. If omitted, uses the current working branch.", + "type": "string" + }, + "labels": { + "description": "Labels to categorize the PR (e.g., 'enhancement', 'bugfix'). Labels must exist in the repository.", + "items": { + "type": "string" + }, + "type": "array" + }, + "title": { + "description": "Concise PR title describing the changes. Follow repository conventions (e.g., conventional commits). The title appears as the main heading.", + "type": "string" + } + }, + "required": [ + "title", + "body" + ], + "type": "object" + }, + "name": "create_pull_request" + }, + { + "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "reason": { + "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", + "type": "string" + }, + "tool": { + "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", + "type": "string" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "name": "missing_tool" + }, + { + "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "message": { + "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "name": "noop" + }, + { + "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "context": { + "description": "Additional context about the missing data or where it should come from (max 256 characters).", + "type": "string" + }, + "data_type": { + "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", + "type": "string" + }, + "reason": { + "description": "Explanation of why this data is needed to complete the task (max 256 characters).", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "name": "missing_data" + } + ] + GH_AW_SAFE_OUTPUTS_TOOLS_EOF + cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' + { + "create_pull_request": { + "defaultMax": 1, + "fields": { + "body": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "branch": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "labels": { + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, + "title": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "tool": { + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "noop": { + "defaultMax": 1, + "fields": { + "message": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + } + } + } + } + GH_AW_SAFE_OUTPUTS_VALIDATION_EOF + - name: Generate Safe Outputs MCP Server Config + id: safe-outputs-config + run: | + # Generate a secure random API key (360 bits of entropy, 40+ chars) + # Mask immediately to prevent timing vulnerabilities + API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${API_KEY}" + + PORT=3001 + + # Set outputs for next steps + { + echo "safe_outputs_api_key=${API_KEY}" + echo "safe_outputs_port=${PORT}" + } >> "$GITHUB_OUTPUT" + + echo "Safe Outputs MCP server will run on port ${PORT}" + + - name: Start Safe Outputs MCP HTTP Server + id: safe-outputs-start + env: + DEBUG: '*' + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + run: | + # Environment variables are set above to prevent template injection + export DEBUG + export GH_AW_SAFE_OUTPUTS_PORT + export GH_AW_SAFE_OUTPUTS_API_KEY + export GH_AW_SAFE_OUTPUTS_TOOLS_PATH + export GH_AW_SAFE_OUTPUTS_CONFIG_PATH + export GH_AW_MCP_LOG_DIR + + bash /opt/gh-aw/actions/start_safe_outputs_server.sh + + - name: Start MCP gateway + id: start-mcp-gateway + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} + GITHUB_MCP_LOCKDOWN: ${{ steps.determine-automatic-lockdown.outputs.lockdown == 'true' && '1' || '0' }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + run: | + set -eo pipefail + mkdir -p /tmp/gh-aw/mcp-config + + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="80" + export MCP_GATEWAY_DOMAIN="host.docker.internal" + MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_API_KEY}" + export MCP_GATEWAY_API_KEY + export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" + mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export DEBUG="*" + + export GH_AW_ENGINE="copilot" + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.4' + + mkdir -p /home/runner/.copilot + cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway.sh + { + "mcpServers": { + "github": { + "type": "stdio", + "container": "ghcr.io/github/github-mcp-server:v0.30.3", + "env": { + "GITHUB_LOCKDOWN_MODE": "$GITHUB_MCP_LOCKDOWN", + "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", + "GITHUB_READ_ONLY": "1", + "GITHUB_TOOLSETS": "repos,issues,pull_requests" + } + }, + "safeoutputs": { + "type": "http", + "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", + "headers": { + "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + } + } + }, + "gateway": { + "port": $MCP_GATEWAY_PORT, + "domain": "${MCP_GATEWAY_DOMAIN}", + "apiKey": "${MCP_GATEWAY_API_KEY}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" + } + } + GH_AW_MCP_CONFIG_EOF + - name: Generate workflow overview + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); + await generateWorkflowOverview(core); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_INPUTS_NAME: ${{ github.event.inputs.name }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + run: | + bash /opt/gh-aw/actions/create_prompt_first.sh + cat << 'GH_AW_PROMPT_EOF' > "$GH_AW_PROMPT" + + GH_AW_PROMPT_EOF + { + cat "/opt/gh-aw/prompts/xpia.md" + cat "/opt/gh-aw/prompts/temp_folder_prompt.md" + cat "/opt/gh-aw/prompts/markdown.md" + cat << 'GH_AW_PROMPT_EOF' + + GitHub API Access Instructions + + The gh CLI is NOT authenticated. Do NOT use gh commands for GitHub operations. + + + To create or modify GitHub resources (issues, discussions, pull requests, etc.), you MUST call the appropriate safe output tool. Simply writing content will NOT work - the workflow requires actual tool calls. + + Temporary IDs: Some safe output tools support a temporary ID field (usually named temporary_id) so you can reference newly-created items elsewhere in the SAME agent output (for example, using #aw_abc1 in a later body). + + **IMPORTANT - temporary_id format rules:** + - If you DON'T need to reference the item later, OMIT the temporary_id field entirely (it will be auto-generated if needed) + - If you DO need cross-references/chaining, you MUST match this EXACT validation regex: /^aw_[A-Za-z0-9]{3,8}$/i + - Format: aw_ prefix followed by 3 to 8 alphanumeric characters (A-Z, a-z, 0-9, case-insensitive) + - Valid alphanumeric characters: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 + - INVALID examples: aw_ab (too short), aw_123456789 (too long), aw_test-id (contains hyphen), aw_id_123 (contains underscore) + - VALID examples: aw_abc, aw_abc1, aw_Test123, aw_A1B2C3D4, aw_12345678 + - To generate valid IDs: use 3-8 random alphanumeric characters or omit the field to let the system auto-generate + + Do NOT invent other aw_* formats — downstream steps will reject them with validation errors matching against /^aw_[A-Za-z0-9]{3,8}$/i. + + Discover available tools from the safeoutputs MCP server. + + **Critical**: Tool calls write structured data that downstream jobs process. Without tool calls, follow-up actions will be skipped. + + **Note**: If you made no other safe output tool calls during this workflow execution, call the "noop" tool to provide a status message indicating completion or that no actions were needed. + + + + The following GitHub context information is available for this workflow: + {{#if __GH_AW_GITHUB_ACTOR__ }} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if __GH_AW_GITHUB_REPOSITORY__ }} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if __GH_AW_GITHUB_WORKSPACE__ }} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} + - **issue-number**: #__GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} + - **discussion-number**: #__GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} + - **pull-request-number**: #__GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} + - **comment-id**: __GH_AW_GITHUB_EVENT_COMMENT_ID__ + {{/if}} + {{#if __GH_AW_GITHUB_RUN_ID__ }} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' + {{#runtime-import .github/workflows/upgrade-pylib.md}} + GH_AW_PROMPT_EOF + } >> "$GH_AW_PROMPT" + - name: Substitute placeholders + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_INPUTS_NAME: ${{ github.event.inputs.name }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + with: + script: | + const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_ENV_ISSUE_ID: process.env.GH_AW_ENV_ISSUE_ID, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, + GH_AW_GITHUB_EVENT_INPUTS_NAME: process.env.GH_AW_GITHUB_EVENT_INPUTS_NAME, + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE + } + }); + - name: Interpolate variables and render templates + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }} + GH_AW_GITHUB_EVENT_INPUTS_NAME: ${{ github.event.inputs.name }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + await main(); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/validate_prompt_placeholders.sh + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/print_prompt_summary.sh + - name: Clean git credentials + run: bash /opt/gh-aw/actions/clean_git_credentials.sh + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 45 + run: | + set -o pipefail + sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains '*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crates.io,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,index.crates.io,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,sh.rustup.rs,static.crates.io,static.rust-lang.org,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.16.4 --skip-pull \ + -- '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_AGENT_COPILOT:+ --model "$GH_AW_MODEL_AGENT_COPILOT"}' \ + 2>&1 | tee /tmp/gh-aw/agent-stdio.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json + GH_AW_MODEL_AGENT_COPILOT: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_WORKSPACE: ${{ github.workspace }} + XDG_CONFIG_HOME: /home/runner + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Copy Copilot session state files to logs + if: always() + continue-on-error: true + run: | + # Copy Copilot session state files to logs folder for artifact collection + # This ensures they are in /tmp/gh-aw/ where secret redaction can scan them + SESSION_STATE_DIR="$HOME/.copilot/session-state" + LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" + if [ -d "$SESSION_STATE_DIR" ]; then + echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" + mkdir -p "$LOGS_DIR" + cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true + echo "Session state files copied successfully" + else + echo "No session-state directory found at $SESSION_STATE_DIR" + fi + - name: Stop MCP gateway + if: always() + continue-on-error: true + env: + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} + run: | + bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" + - name: Redact secrets in logs + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + await main(); + env: + GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + SECRET_COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Safe Outputs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: safe-output + path: ${{ env.GH_AW_SAFE_OUTPUTS }} + if-no-files-found: warn + - name: Ingest agent output + id: collect_output + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crates.io,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,index.crates.io,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,sh.rustup.rs,static.crates.io,static.rust-lang.org,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + await main(); + - name: Upload sanitized agent output + if: always() && env.GH_AW_AGENT_OUTPUT + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent-output + path: ${{ env.GH_AW_AGENT_OUTPUT }} + if-no-files-found: warn + - name: Upload engine output files + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent_outputs + path: | + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log + if-no-files-found: ignore + - name: Parse agent logs for step summary + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + await main(); + - name: Parse MCP gateway logs for step summary + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Print firewall logs + if: always() + continue-on-error: true + env: + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs + run: | + # Fix permissions on firewall logs so they can be uploaded as artifacts + # AWF runs with sudo, creating files owned by root + sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || true + awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent-artifacts + path: | + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/mcp-logs/ + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/agent/ + /tmp/gh-aw/aw*.patch + if-no-files-found: ignore + + conclusion: + needs: + - activation + - agent + - detection + - safe_outputs + if: (always()) && (needs.agent.result != 'skipped') + runs-on: ubuntu-slim + permissions: + contents: read + discussions: write + issues: write + pull-requests: write + outputs: + noop_message: ${{ steps.noop.outputs.noop_message }} + tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} + total_count: ${{ steps.missing_tool.outputs.total_count }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@ff62cdbec36230acbae869ddb28806e8eca01ea1 # v0.87.10 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Process No-Op Messages + id: noop + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: 1 + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/noop.cjs'); + await main(); + - name: Record Missing Tool + id: missing_tool + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + await main(); + - name: Handle Agent Failure + id: handle_agent_failure + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_WORKFLOW_ID: "upgrade-pylib" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.agent.outputs.secret_verification_result }} + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + await main(); + - name: Handle No-Op Message + id: handle_noop_message + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + await main(); + - name: Handle Create Pull Request Error + id: handle_create_pr_error + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_create_pr_error.cjs'); + await main(); + - name: Update reaction comment with completion status + id: conclusion + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_COMMENT_REPO: ${{ needs.activation.outputs.comment_repo }} + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.result }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/notify_comment_error.cjs'); + await main(); + + detection: + needs: agent + if: needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true' + runs-on: ubuntu-latest + permissions: {} + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" + timeout-minutes: 10 + outputs: + success: ${{ steps.parse_results.outputs.success }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@ff62cdbec36230acbae869ddb28806e8eca01ea1 # v0.87.10 + with: + destination: /opt/gh-aw/actions + - name: Download agent artifacts + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent-artifacts + path: /tmp/gh-aw/threat-detection/ + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent-output + path: /tmp/gh-aw/threat-detection/ + - name: Echo agent output types + env: + AGENT_OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + run: | + echo "Agent output-types: $AGENT_OUTPUT_TYPES" + - name: Setup threat detection + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + WORKFLOW_NAME: "Upgrade Python Library" + WORKFLOW_DESCRIPTION: "Pick an out-of-sync Python library from the todo list and upgrade it\nby running `scripts/update_lib quick`, then open a pull request." + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + await main(); + - name: Ensure threat-detection directory and log + run: | + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Install GitHub Copilot CLI + run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.409 + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + # --allow-tool shell(cat) + # --allow-tool shell(grep) + # --allow-tool shell(head) + # --allow-tool shell(jq) + # --allow-tool shell(ls) + # --allow-tool shell(tail) + # --allow-tool shell(wc) + timeout-minutes: 20 + run: | + set -o pipefail + COPILOT_CLI_INSTRUCTION="$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" + mkdir -p /tmp/ + mkdir -p /tmp/gh-aw/ + mkdir -p /tmp/gh-aw/agent/ + mkdir -p /tmp/gh-aw/sandbox/agent/logs/ + copilot --add-dir /tmp/ --add-dir /tmp/gh-aw/ --add-dir /tmp/gh-aw/agent/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-tool 'shell(cat)' --allow-tool 'shell(grep)' --allow-tool 'shell(head)' --allow-tool 'shell(jq)' --allow-tool 'shell(ls)' --allow-tool 'shell(tail)' --allow-tool 'shell(wc)' --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$COPILOT_CLI_INSTRUCTION"${GH_AW_MODEL_DETECTION_COPILOT:+ --model "$GH_AW_MODEL_DETECTION_COPILOT"} 2>&1 | tee /tmp/gh-aw/threat-detection/detection.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_MODEL_DETECTION_COPILOT: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_WORKSPACE: ${{ github.workspace }} + XDG_CONFIG_HOME: /home/runner + - name: Parse threat detection results + id: parse_results + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + - name: Upload threat detection log + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: threat-detection.log + path: /tmp/gh-aw/threat-detection/detection.log + if-no-files-found: ignore + + safe_outputs: + needs: + - activation + - agent + - detection + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.detection.outputs.success == 'true') + runs-on: ubuntu-slim + permissions: + contents: write + issues: write + pull-requests: write + timeout-minutes: 15 + env: + GH_AW_ENGINE_ID: "copilot" + GH_AW_WORKFLOW_ID: "upgrade-pylib" + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + outputs: + create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} + create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} + process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@ff62cdbec36230acbae869ddb28806e8eca01ea1 # v0.87.10 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Download patch artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent-artifacts + path: /tmp/gh-aw/ + - name: Checkout repository + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (contains(needs.agent.outputs.output_types, 'create_pull_request')) + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + token: ${{ github.token }} + persist-credentials: false + fetch-depth: 1 + - name: Configure Git credentials + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (contains(needs.agent.outputs.output_types, 'create_pull_request')) + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + GIT_TOKEN: ${{ github.token }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${GIT_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_pull_request\":{\"base_branch\":\"${{ github.ref_name }}\",\"draft\":false,\"expires\":30,\"labels\":[\"pylib-sync\"],\"max\":1,\"max_patch_size\":1024,\"title_prefix\":\"Update \"},\"missing_data\":{},\"missing_tool\":{}}" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + await main(); + diff --git a/.github/workflows/upgrade-pylib.md b/.github/workflows/upgrade-pylib.md new file mode 100644 index 00000000000..cd05eb77734 --- /dev/null +++ b/.github/workflows/upgrade-pylib.md @@ -0,0 +1,134 @@ +--- +description: | + Pick an out-of-sync Python library from the todo list and upgrade it + by running `scripts/update_lib quick`, then open a pull request. + +on: + workflow_dispatch: + inputs: + name: + description: "Module name to upgrade (leave empty to auto-pick)" + required: false + type: string + +timeout-minutes: 45 + +permissions: + contents: read + issues: read + pull-requests: read + +network: + allowed: + - defaults + - rust + - python + +engine: copilot + +runtimes: + python: + version: "3.14" + +tools: + bash: + - ":*" + edit: + github: + toolsets: [repos, issues, pull_requests] + read-only: true + +safe-outputs: + create-pull-request: + title-prefix: "Update " + labels: [pylib-sync] + draft: false + expires: 30 + +cache: + key: cpython-lib-${{ env.PYTHON_VERSION }} + path: cpython + restore-keys: + - cpython-lib- + +env: + PYTHON_VERSION: "v3.14.7" + ISSUE_ID: "6839" +--- + +# Upgrade Python Library + +You are an automated maintenance agent for RustPython, a Python 3 interpreter written in Rust. Your task is to upgrade one out-of-sync Python standard library module from CPython. + +## Step 1: Set up the environment + +The CPython source may already be cached. Check if the `cpython` directory exists and has the correct version: + +```bash +if [ -d "cpython/Lib" ]; then + echo "CPython cache hit, skipping clone" +else + git clone --depth 1 --branch "$PYTHON_VERSION" https://github.com/python/cpython.git cpython +fi +``` + +## Step 2: Determine module name + +Run this script to determine the module name: + +```bash +MODULE_NAME="${{ github.event.inputs.name }}" +if [ -z "$MODULE_NAME" ]; then + echo "No module specified, running todo to find one..." + python3 scripts/update_lib todo + echo "Pick one module from the list above that is marked [ ], has no unmet deps, and has a small Δ number." + echo "Do NOT pick: opcode, datetime, random, hashlib, tokenize, pdb, _pyrepl, concurrent, asyncio, multiprocessing, ctypes, idlelib, tkinter, shutil, tarfile, email, unittest" +else + echo "Module specified by user: $MODULE_NAME" +fi +``` + +If the script printed "Module specified by user: ...", use that exact name. If it printed the todo list, pick one suitable module from it. + +## Step 3: Run the upgrade + +Run the quick upgrade command. This will copy the library from CPython, migrate test files preserving RustPython markers, auto-mark test failures, and create a git commit: + +```bash +python3 scripts/update_lib quick +``` + +This takes a while because it builds RustPython (`cargo build --release`) and runs tests to determine which ones pass or fail. + +If the command fails, report the error and stop. Do not try to fix Rust code or modify test files manually. + +## Step 4: Verify the result + +After the script succeeds, check what changed: + +```bash +git log -1 --stat +git diff HEAD~1 --stat +``` + +Make sure the commit was created with the correct message format: `Update from `. + +## Step 5: Create the pull request + +Create a pull request. Reference issue #${{ env.ISSUE_ID }} in the body but do **NOT** use keywords that auto-close issues (Fix, Close, Resolve). + +Use this format for the PR body: + +``` +## Summary + +Upgrade `` from CPython $PYTHON_VERSION. + +Part of #$ISSUE_ID + +## Changes + +- Updated `Lib/` from CPython +- Migrated test files preserving RustPython markers +- Auto-marked test failures with `@expectedFailure` +``` diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 00000000000..43d8f16039c --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,29 @@ +rules: + dangerous-triggers: + ignore: + # pull_request_target is needed to label and assign PRs from forks with issues: write. + # The workflow does not check out or execute pull request code. + - oscca-pr.yml:3 + # workflow_run is needed to post a PR comment from a fork's run, which is + # handed a read-only token however the triggering workflow is configured. + # The workflow does not check out or execute pull request code: it reads + # one artifact and writes a comment. + - pyperformance-comment.yaml:1 + excessive-permissions: + ignore: + # pull_request_target is needed to post PR comments with pull-requests: write. + # Workflow-level permissions: {} restricts defaults; only the job has write access. + - lib-deps-check.yaml:3 + unpinned-uses: + config: + policies: + # dtolnay/rust-toolchain is a trusted action that uses lightweight branch + # refs (@stable, @nightly, etc.) by design. Pinning to a hash would break + # the intended usage pattern. + # We can remove this once https://github.com/dtolnay/rust-toolchain/issues/180 is resolved + dtolnay/rust-toolchain: any + # dtolnay/rust-toolchain handles component installation, target addition, and + # override configuration beyond what a bare `rustup` invocation provides. + # See: https://github.com/zizmorcore/zizmor/issues/1817 + superfluous-actions: + disable: true diff --git a/.gitignore b/.gitignore index c03ae1a997e..cb22084d3dd 100644 --- a/.gitignore +++ b/.gitignore @@ -8,13 +8,15 @@ __pycache__/ .repl_history.txt .vscode/ wasm-pack.log -.idea/ +.idea/* +!.idea/icon.svg +!.idea/vcs.xml .envrc -.python-version flame-graph.html flame.txt flamescope.json +/build/codspeed-environment*.json /wapm.lock /wapm_packages /.cargo/config @@ -22,9 +24,15 @@ flamescope.json extra_tests/snippets/resources extra_tests/not_impl.py +Lib/_sysconfig_vars*.json Lib/site-packages/* !Lib/site-packages/README.txt Lib/test/data/* !Lib/test/data/README cpython/ +.claude/ +docs/superpowers/ +scripts/pyperformance/results/ +scripts/pyperformance/stub_psutil/build/ +scripts/pyperformance/stub_psutil/src/psutil.egg-info/ \ No newline at end of file diff --git a/.idea/icon.svg b/.idea/icon.svg new file mode 100644 index 00000000000..84e5f593a6d --- /dev/null +++ b/.idea/icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000000..82bc0911775 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,16 @@ + + + + + + + + + \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000..117ee6f9339 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,84 @@ +# NOTE: Reason for not using `prek.toml` is dependabot supports `pre-commit` as an ecosystem +# See: https://github.blog/changelog/2026-03-10-dependabot-now-supports-pre-commit-hooks/ + +fail_fast: false +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-merge-conflict + priority: 0 + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.4 + hooks: + - id: ruff-format + priority: 0 + + - id: ruff-check + args: [--select, I, --fix, --exit-non-zero-on-fix] + types_or: [python] + require_serial: true + priority: 1 + + - repo: local + hooks: + - id: redundant-test-patches + name: check redundant test patches + entry: scripts/check_redundant_patches.py + files: '^Lib/test/.*\.py$' + language: script + types: [python] + priority: 0 + + - repo: local + hooks: + - id: rustfmt + name: rustfmt + entry: rustfmt + language: system + types: [rust] + priority: 0 + + - id: generate-rs-opcode-metadata + name: generate rust opcode metadata + entry: python3 tools/opcode_metadata/generate_rs_opcode_metadata.py + files: '^(crates/compiler-core/src/bytecode/instruction\.rs|tools/opcode_metadata/*)$' + pass_filenames: false + language: system + require_serial: true + priority: 1 # so rustfmt runs first + stages: + - manual + + - id: generate-py-opcode-metadata + name: generate python opcode metadata + entry: python3 tools/opcode_metadata/generate_py_opcode_metadata.py + files: '^(crates/compiler-core/src/bytecode/instruction\.rs|tools/opcode_metadata/*)$' + pass_filenames: false + language: system + require_serial: true + priority: 1 # so rustfmt runs first + stages: + - manual + + - repo: https://github.com/streetsidesoftware/cspell-cli + rev: v10.1.0 + hooks: + - id: cspell + types: [rust] + additional_dependencies: + - '@cspell/dict-en_us' + - '@cspell/dict-cpp' + - '@cspell/dict-python' + - '@cspell/dict-rust' + - '@cspell/dict-win32' + - '@cspell/dict-shell' + priority: 0 + + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.9.6 + hooks: + - id: prettier + files: '^wasm/.*$' + priority: 0 diff --git a/.python-version b/.python-version new file mode 100644 index 00000000000..a128d5c0d97 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14.7 diff --git a/AGENTS.md b/AGENTS.md index fa14977953a..80a79d61cde 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ -# GitHub Copilot Instructions for RustPython +# AI Agent Instructions for RustPython -This document provides guidelines for working with GitHub Copilot when contributing to the RustPython project. +This document provides guidelines for AI coding agents (GitHub Copilot, Claude Code, Gemini, etc.) contributing to the RustPython project. ## Project Overview @@ -13,31 +13,27 @@ RustPython is a Python 3 interpreter written in Rust, implementing Python 3.14.0 ## Repository Structure -- `src/` - Top-level code for the RustPython binary -- `vm/` - The Python virtual machine implementation - - `builtins/` - Python built-in types and functions - - `stdlib/` - Essential standard library modules implemented in Rust, required to run the Python core -- `compiler/` - Python compiler components - - `parser/` - Parser for converting Python source to AST - - `core/` - Bytecode representation in Rust structures - - `codegen/` - AST to bytecode compiler -- `Lib/` - CPython's standard library in Python (copied from CPython). **IMPORTANT**: Do not edit this directory directly; The only allowed operation is copying files from CPython. -- `derive/` - Rust macros for RustPython -- `common/` - Common utilities -- `extra_tests/` - Integration tests and snippets -- `stdlib/` - Non-essential Python standard library modules implemented in Rust (useful but not required for core functionality) -- `wasm/` - WebAssembly support -- `jit/` - Experimental JIT compiler implementation -- `pylib/` - Python standard library packaging (do not modify this directory directly - its contents are generated automatically) +See the "Code organization" section in [CONTRIBUTING.md](CONTRIBUTING.md#code-organization) for the current directory layout. ## AI Agent Rules +**CRITICAL: AI Policy** + +- Follow RustPython's [AI Policy](https://github.com/RustPython/.github/blob/main/AI_POLICY.md) for every AI-assisted contribution. +- Disclose AI assistance in commit messages with an `Assisted-by: AGENT_NAME:MODEL_VERSION` trailer. Use one trailer per AI tool, and never use `Co-authored-by` for an AI assistant. + **CRITICAL: Git Operations** - NEVER create pull requests directly without explicit user permission - NEVER push commits to remote without explicit user permission - Always ask the user before performing any git operations that affect the remote repository - Commits can be created locally when requested, but pushing and PR creation require explicit approval +**CRITICAL: Commit Hooks and Validation** +- Install the repository's pre-commit hook with `prek install` (or `pre-commit install`) after cloning the repository. +- Every commit must run the configured pre-commit hook. NEVER bypass it with `--no-verify`. Automated workflows that use a normal `git commit`, such as `scripts/update_lib quick`, should be allowed to create local commits through the hook. +- If a hook auto-fixes files (e.g. `ruff-format`, `rustfmt`), re-stage the fixes and retry the commit. Do not amend or force a failing commit through. +- Before completing a task, run the tests appropriate for the change. Test commands are documented in the [Testing](#testing) section below. At minimum run `cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher --exclude rustpython-capi`, then run `cargo test` from `crates/capi`; if the change touches `extra_tests/snippets/` run `pytest -v` there too, and if it touches `Lib/` or interpreter behavior, run the relevant `cargo run --release -- -m test ` modules. + ## Important Development Notes ### Running Python Code @@ -81,6 +77,35 @@ The `Lib/` directory contains Python standard library files copied from the CPyt - `unittest.skip("TODO: RustPython ")` - `unittest.expectedFailure` with `# TODO: RUSTPYTHON ` comment +#### Choosing the right marker + +When marking a test that fails on RustPython, prefer one of the following forms: + +```python +@unittest.expectedFailure # TODO: RUSTPYTHON; +# or +@unittest.expectedFailureIf(, "TODO: RUSTPYTHON; ") +``` + +If the test would crash the interpreter (segfault, Rust panic, abort, infinite loop), use `skip` instead so the rest of the suite can still run: + +```python +@unittest.skip("TODO: RUSTPYTHON; ") +# or +@unittest.skipIf(, "TODO: RUSTPYTHON; ") +``` + +**When to use which:** + +- **Prefer `expectedFailure` / `expectedFailureIf`** by default. The test body still runs, so if RustPython is later fixed, the unexpected pass surfaces immediately and the decorator can be removed. Use the conditional `*If` form when the failure is environment-specific (e.g., a platform or build flag). +- **Use `skip` / `skipIf` only when running the test would take down the test process** — segfaults, Rust panics, aborts, or hangs that block subsequent tests. Skipping keeps the suite usable; `expectedFailure` cannot help here, because the test body still executes. + +To find WIP entries that are partly modified and may need follow-up: + +```bash +grep -d recurse 'TODO: RUSTPYTHON' Lib/test/ +``` + ### Clean Build When you modify bytecode instructions, a full clean is required: @@ -93,7 +118,14 @@ rm -r target/debug/build/rustpython-* && find . | grep -E "\.pyc$" | xargs rm -r ```bash # Run Rust unit tests -cargo test --workspace --exclude rustpython_wasm +cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher --exclude rustpython-capi + +# Run C-API tests from their directory so their separate Cargo config applies +(cd crates/capi && cargo test) + +# Run Clippy with the same C-API separation +cargo clippy --workspace --all-targets --exclude rustpython_wasm --exclude rustpython-venvlauncher --exclude rustpython-capi +(cd crates/capi && cargo clippy --all-targets) # Run Python snippets tests (debug mode recommended for faster compilation) cargo run -- extra_tests/snippets/builtin_bytes.py @@ -125,6 +157,37 @@ Run `./scripts/whats_left.py` to get a list of unimplemented methods, which is h - Follow Rust best practices for error handling and memory management - Use the macro system (`pyclass`, `pymodule`, `pyfunction`, etc.) when implementing Python functionality in Rust +#### Comments + +- Do not delete or rewrite existing comments unless they are factually wrong or directly contradict the new code. +- Do not add decorative section separators (e.g. `// -----------`, `// ===`, `/* *** */`). Use `///` doc-comments or short `//` comments only when they add value. +- Do not put `///` doc comments on items annotated with `#[pyattr]`, `#[pyclass]`, or `#[pyfunction]`. The derive macros pull authoritative docstrings from CPython via the `rustpython-doc` crate; a Rust doc comment overrides that source, and on `#[pyattr]` it is silently dropped. + +#### Avoid Duplicate Code in Branches + +When branches differ only in a value but share common logic, extract the differing value first, then call the common logic once. + +**Bad:** +```rust +let result = if condition { + let msg = format!("message A: {x}"); + some_function(msg, shared_arg) +} else { + let msg = format!("message B"); + some_function(msg, shared_arg) +}; +``` + +**Good:** +```rust +let msg = if condition { + format!("message A: {x}") +} else { + format!("message B") +}; +let result = some_function(msg, shared_arg); +``` + ### Python Code - **IMPORTANT**: In most cases, Python code should not be edited. Bug fixes should be made through Rust code modifications only @@ -197,9 +260,11 @@ cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdli cargo run --features jit ``` -### Building venvlauncher (Windows) +### Linux Build and Debug on macOS -See DEVELOPMENT.md "CPython Version Upgrade Checklist" section. +See the "Testing on Linux from macOS" section in [CONTRIBUTING.md](CONTRIBUTING.md#testing-on-linux-from-macos). + +### Building venvlauncher (Windows) **IMPORTANT**: All 4 venvlauncher binaries use the same source code. Do NOT add multiple `[[bin]]` entries to Cargo.toml. Build once and copy with different names. @@ -224,9 +289,14 @@ See DEVELOPMENT.md "CPython Version Upgrade Checklist" section. - Document that it requires PEP 695 support - Focus on tests that can be fixed through Rust code changes only +## CI Workflows + +If you modify any file under `.github/workflows/`, the change must pass a [zizmor](https://docs.zizmor.sh/) scan in CI. + ## Documentation - Check the [architecture document](/architecture/architecture.md) for a high-level overview -- Read the [development guide](/DEVELOPMENT.md) for detailed setup instructions +- Read the [development guide](/CONTRIBUTING.md) for detailed setup instructions - Generate documentation with `cargo doc --no-deps --all` - Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/) +- [How to update test files](https://github.com/RustPython/RustPython/wiki/How-to-update-test-files#checkout-cpython-source-code-initial-setup) — guide for syncing test cases from upstream CPython into the `Lib/` directory diff --git a/DEVELOPMENT.md b/CONTRIBUTING.md similarity index 85% rename from DEVELOPMENT.md rename to CONTRIBUTING.md index 24c149eebef..637ecf0993a 100644 --- a/DEVELOPMENT.md +++ b/CONTRIBUTING.md @@ -1,4 +1,28 @@ -# RustPython Development Guide and Tips +# Contributing to RustPython + +Contributions are more than welcome, and in many cases we are happy to guide +contributors through PRs or on [**Discord**](https://discord.gg/vru8NypEhv). + +## Finding ways to help + +We label issues that would be good for a first time contributor as [`good first issue`](https://github.com/RustPython/RustPython/issues?q=label%3A%22good+first+issue%22+is%3Aissue+is%3Aopen+). +Also checkout the [issue tracker](https://github.com/RustPython/RustPython/issues) for all open issues. + +You can enhance CPython compatibility by increasing our unittest coverage, you can see [This pinned issue](https://github.com/RustPython/RustPython/issues/6839) to see which libs and tests need be updated to our current supported python version. + +Another approach is to checkout the source code: builtin functions and object +methods are often the simplest and easiest way to contribute. + +You can also simply run `python -I scripts/whats_left.py` to assist in finding any unimplemented method. + +## Use of AI + +We **require all use of AI in contributions to follow our +[AI Policy](https://github.com/RustPython/.github/blob/main/AI_POLICY.md)**. + +If your contribution does not follow the policy, it will be closed. + +## RustPython Development Guide and Tips RustPython attracts developers with interest and experience in Rust, Python, or WebAssembly. Whether you are familiar with Rust, Python, or @@ -65,7 +89,15 @@ $ pytest -v Rust unit tests can be run with `cargo`: ```shell -$ cargo test --workspace --exclude rustpython_wasm +$ cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher --exclude rustpython-capi +``` + +`rustpython-capi` needs to be tested from inside its own directory, since it has a +separate `cargo` config that only applies there: + +```shell +$ cd crates/capi +$ cargo test ``` Python unit tests can be run by compiling RustPython and running the test module: diff --git a/Cargo.lock b/Cargo.lock index c6462e235f3..0cbc19fbe68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,40 +2,45 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "adler32" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" +[[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "aes" -version = "0.8.4" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" dependencies = [ - "cfg-if", "cipher", + "cpubits", "cpufeatures", ] [[package]] -name = "ahash" -version = "0.8.12" +name = "aes-gcm" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ - "cfg-if", - "getrandom 0.3.4", - "once_cell", - "version_check", - "zerocopy", + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", ] [[package]] @@ -47,30 +52,12 @@ dependencies = [ "memchr", ] -[[package]] -name = "alloca" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" -dependencies = [ - "cc", -] - [[package]] name = "allocator-api2" version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - [[package]] name = "anes" version = "0.1.6" @@ -79,9 +66,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -94,15 +81,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -129,9 +116,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "approx" @@ -144,9 +131,9 @@ dependencies = [ [[package]] name = "ar_archive_writer" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348" dependencies = [ "object", ] @@ -157,6 +144,12 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "ascii" version = "1.1.0" @@ -165,9 +158,9 @@ checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "asn1-rs" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -175,7 +168,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 2.0.18", + "thiserror", "time", ] @@ -187,7 +180,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -199,7 +192,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -222,7 +215,7 @@ dependencies = [ "manyhow", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -238,58 +231,59 @@ dependencies = [ "proc-macro2", "quote", "quote-use", - "syn", + "syn 2.0.119", ] [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-fips-sys" -version = "0.13.11" +version = "0.13.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6ea8e07e2df15b9f09f2ac5ee2977369b06d116f0c4eb5fa4ad443b73c7f53" +checksum = "6c0e6249c249b8916c98ebae7bc06216c8dcab3002f32872b4abe642d17063b1" dependencies = [ "bindgen 0.72.1", "cc", "cmake", "dunce", "fs_extra", + "pkg-config", "regex", ] [[package]] name = "aws-lc-rs" -version = "1.15.4" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" dependencies = [ "aws-lc-fips-sys", "aws-lc-sys", - "untrusted 0.7.1", "zeroize", ] [[package]] name = "aws-lc-sys" -version = "0.37.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c34dda4df7017c8db52132f0f8a2e0f8161649d15723ed63fc00c82d0f2081a" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] name = "base64" -version = "0.22.1" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" [[package]] name = "base64ct" @@ -303,7 +297,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -313,8 +307,8 @@ dependencies = [ "quote", "regex", "rustc-hash", - "shlex", - "syn", + "shlex 1.3.0", + "syn 2.0.119", ] [[package]] @@ -323,7 +317,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -333,8 +327,8 @@ dependencies = [ "quote", "regex", "rustc-hash", - "shlex", - "syn", + "shlex 1.3.0", + "syn 2.0.119", ] [[package]] @@ -345,68 +339,85 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] -name = "blake2" -version = "0.10.6" +name = "bitflagset" +version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "64b6ee310aa7af14142c8c9121775774ff601ae055ed98ba7fac96098bcde1b9" dependencies = [ - "digest", + "num-integer", + "num-traits", + "radium", + "ref-cast", +] + +[[package]] +name = "blake2b_simd" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3560a7b1951efe814fcd721938313adc56753ca39f4b23847d7e9a2402f5dbff" +dependencies = [ + "arrayvec", + "constant_time_eq 0.4.2", +] + +[[package]] +name = "blake2s_simd" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380c0236432f7b22a70229df30f218f5293870c056beb76f5ca068e3273a366" +dependencies = [ + "arrayvec", + "constant_time_eq 0.4.2", ] [[package]] name = "block-buffer" -version = "0.10.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] name = "block-padding" -version = "0.3.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] name = "bstr" -version = "1.12.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" dependencies = [ "memchr", "regex-automata", - "serde", + "serde_core", ] [[package]] name = "bumpalo" -version = "3.19.1" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" dependencies = [ "allocator-api2", ] [[package]] name = "bytemuck" -version = "1.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" - -[[package]] -name = "bytes" -version = "1.11.0" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" [[package]] name = "bzip2" @@ -417,15 +428,6 @@ dependencies = [ "libbz2-rs-sys", ] -[[package]] -name = "caseless" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6fd507454086c8edfd769ca6ada439193cdb209c7681712ef6275cccbfe5d8" -dependencies = [ - "unicode-normalization", -] - [[package]] name = "cast" version = "0.3.0" @@ -443,31 +445,25 @@ dependencies = [ [[package]] name = "cbc" -version = "0.1.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" dependencies = [ "cipher", ] [[package]] name = "cc" -version = "1.2.54" +version = "1.2.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" +checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex", + "shlex 2.0.1", ] -[[package]] -name = "cesu8" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" - [[package]] name = "cexpr" version = "0.6.0" @@ -490,16 +486,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] -name = "chrono" -version = "0.4.43" +name = "chacha20" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-link", + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + +[[package]] +name = "char_str" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576ba56f6ca18ebb069d0d07260407171712aff4dfe15ee3f8982dc16a455bcf" +dependencies = [ + "castaway", + "get-size2", + "itoa", + "ryu", ] [[package]] @@ -531,10 +537,11 @@ dependencies = [ [[package]] name = "cipher" -version = "0.4.4" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ + "block-buffer", "crypto-common", "inout", ] @@ -552,18 +559,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.54" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.54" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstyle", "clap_lex", @@ -571,9 +578,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.7" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "clipboard-win" @@ -586,20 +593,75 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] [[package]] -name = "cobs" -version = "0.3.0" +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "codspeed" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7083f253260bcb4aaa3b4aa4c52973703dabc1a85c2f193997e2689aafa8a919" +dependencies = [ + "anyhow", + "cc", + "colored", + "getrandom 0.4.3", + "glob", + "libc", + "nix 0.31.3", + "serde", + "serde_json", + "statrs", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9f24251445188c69d50f10179d795424bd71b533b7e3f84fc03f26893b01af0" +dependencies = [ + "clap", + "codspeed", + "codspeed-criterion-compat-walltime", + "colored", + "regex", +] + +[[package]] +name = "codspeed-criterion-compat-walltime" +version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +checksum = "5c38205d56e2cb4fe04b708de7f9653a3f1b89edbe3a20b28f21e9e525e9e061" dependencies = [ - "thiserror 2.0.18", + "anes", + "cast", + "ciborium", + "clap", + "codspeed", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", ] [[package]] @@ -610,44 +672,41 @@ checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] -name = "combine" -version = "4.6.7" +name = "colored" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "bytes", - "memchr", + "windows-sys 0.61.2", ] [[package]] name = "compact_str" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" dependencies = [ "castaway", "cfg-if", "itoa", - "rustversion", - "ryu", "static_assertions", + "zmij", ] [[package]] name = "console" -version = "0.15.11" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" dependencies = [ "encode_unicode", "libc", - "once_cell", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -662,9 +721,9 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.6" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" [[package]] name = "constant_time_eq" @@ -673,13 +732,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] -name = "core-foundation" -version = "0.9.4" +name = "constant_time_eq" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "6187e670c346a04bcbf5def0393d57ec0e5ab56565cde6be4a7cbdb6cdc2f3f6" dependencies = [ - "core-foundation-sys", - "libc", + "rustversion", ] [[package]] @@ -698,20 +756,26 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] [[package]] name = "cranelift" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d483a248b5d971d1ef6a814385502a38d8dde8fbf08b4ad08b78c53b8d66f923" +checksum = "4a9ea11e186263f9021825558839b756e8dcb74b63f8f5bb0335b1f93d6ef6d5" dependencies = [ "cranelift-codegen", "cranelift-frontend", @@ -720,46 +784,46 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32b9105ce689b3e79ae288f62e9c2d0de66e4869176a11829e5c696da0f018f" +checksum = "9903e767cc0a95a59950de099a6b1d61e1476f774be1cdfa06d7ccdc9bfae56f" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e950e8dd96c1760f1c3a2b06d3d35584a3617239d034e73593ec096a1f3ea69" +checksum = "e777252ea3ec9daffa71408fbfe0793ab0a7b7bc3b2cf0f417dd0fa33964b8ef" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d769576bc48246fccf7f07173739e5f7a7fb3270eb9ac363c0792cad8963c034" +checksum = "7828208a36e6627481cea61bd24c6234e31411d5c58b48e80ca094a1d593b944" dependencies = [ "cranelift-entity", + "wasmtime-internal-core", ] [[package]] name = "cranelift-bitset" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d37c4589e52def48bd745c3b28b523d66ade8b074644ed3a366144c225f212" +checksum = "a58c8447cc59ef98c49096679f81392ad0acec2224cea2bef52011f5169f9d02" dependencies = [ - "serde", - "serde_derive", + "wasmtime-internal-core", ] [[package]] name = "cranelift-codegen" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c23b5ab93367eba82bddf49b63d841d8a0b8b39fb89d82829de6647b3a747108" +checksum = "ce520e899df1de583e7b564eafd66760fbc147d72894d2278ddac57a1489fcc0" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -771,21 +835,22 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli", - "hashbrown 0.15.5", + "hashbrown 0.17.1", + "libm", "log", "regalloc2", "rustc-hash", "serde", "smallvec", "target-lexicon", - "wasmtime-internal-math", + "wasmtime-internal-core", ] [[package]] name = "cranelift-codegen-meta" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c6118d26dd046455d31374b9432947ea2ba445c21fd8724370edd072f51f3bd" +checksum = "8d8ab23e63d78ea6bbda18ae1ed9c958bc1cb88d90fb5e95f5ba62e9c019474b" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -795,37 +860,37 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a068c67f04f37de835fda87a10491e266eea9f9283d0887d8bd0a2c0726588a9" +checksum = "24319a996b1ef40ebf8af69b39c868c790b35a42cfe38edacfd8c9deb75ea712" [[package]] name = "cranelift-control" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ceb830549fcd7f05493a3b6d3d2bcfa4d43588b099e8c2393d2d140d6f7951" +checksum = "cf9856d68cc2cefb83229cb2c55b620dd38427555c2fe87d73f8a4421616f628" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b130f0edd119e7665f1875b8d686bd3fccefd9d74d10e9005cbcd76392e1831" +checksum = "e0ec225d6b79c5d61358cb5ea081a1384ec541dbf8c1a8c618d187875af326ad" dependencies = [ "cranelift-bitset", - "serde", - "serde_derive", + "wasmtime-internal-core", ] [[package]] name = "cranelift-frontend" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626a46aa207183bae011de3411a40951c494cea3fb2ef223d3118f75e13b23ca" +checksum = "7d3a96a09bd9bd3cf6df2de2ea15d7418f493e946a8eaa1acfe39297a67ed3f3" dependencies = [ "cranelift-codegen", + "hashbrown 0.17.1", "log", "smallvec", "target-lexicon", @@ -833,15 +898,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09dab08a5129cf59919fdd4567e599ea955de62191a852982150ac42ce4ab21" +checksum = "bd28f593c36afbdc9aa17305c0bb66c39ce0530e1caffcf389fbec053950083e" [[package]] name = "cranelift-jit" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaab95b37e712267c51ca968ed4fa83d1a79b9ff3bc86fb9469c764340f486e4" +checksum = "d3e00ac7d32a2662e3685ac3161b9d62050c8cef05f0618eef28cba8e19e808d" dependencies = [ "anyhow", "cranelift-codegen", @@ -851,6 +916,7 @@ dependencies = [ "cranelift-native", "libc", "log", + "memmap2", "region", "target-lexicon", "wasmtime-internal-jit-icache-coherence", @@ -859,9 +925,9 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d53f2d6b64ef9fb21da36698d45715639e0df50224883baa1e9bd04f96f0716" +checksum = "89471cd942ab4d966b8c08a258a44042d98a9151dab90128da794e0ceb057927" dependencies = [ "anyhow", "cranelift-codegen", @@ -870,9 +936,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "847b8eaef0f7095b401d3ce80587036495b94e7a051904df9e28d6cd14e69b94" +checksum = "2987ca47affc261aa31ac643fb80dd7f8274cebf878003ba9f50cd6ff2c1055f" dependencies = [ "cranelift-codegen", "libc", @@ -881,9 +947,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.128.1" +version = "0.135.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15a4849e90e778f2fcc9fd1b93bd074dbf6b8b6f420951f9617c4774fe71e7fc" +checksum = "b7431dd8c9def94aca43b27915dee0d42103d9b13922cfb044febcd975547d68" [[package]] name = "crc32fast" @@ -894,46 +960,21 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "criterion" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d883447757bb0ee46f233e9dc22eb84d93a9508c9b868687b274fc431d886bf" -dependencies = [ - "alloca", - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "itertools 0.13.0", - "num-traits", - "oorandom", - "page_size", - "plotters", - "rayon", - "regex", - "serde", - "serde_json", - "tinytemplate", - "walkdir", -] - [[package]] name = "criterion-plot" -version = "0.8.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed943f81ea2faa8dcecbbfa50164acf95d555afec96a27871663b300e387b2e4" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools 0.13.0", + "itertools 0.10.5", ] [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -941,18 +982,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -962,12 +1003,11 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", ] [[package]] @@ -979,22 +1019,69 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "data-encoding" -version = "2.10.0" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] [[package]] name = "der" -version = "0.7.10" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" dependencies = [ "const-oid", - "der_derive", - "flagset", - "pem-rfc7468 0.7.0", + "pem-rfc7468", "zeroize", ] @@ -1012,121 +1099,96 @@ dependencies = [ "rusticata-macros", ] -[[package]] -name = "der_derive" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "deranged" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" -dependencies = [ - "powerfmt", -] +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" [[package]] name = "derive-where" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "digest" -version = "0.10.7" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer", + "const-oid", "crypto-common", - "subtle", + "ctutils", ] [[package]] -name = "dirs-next" -version = "2.0.0" +name = "dirs" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "cfg-if", - "dirs-sys-next", + "dirs-sys", ] [[package]] -name = "dirs-sys-next" -version = "0.1.2" +name = "dirs-sys" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", + "option-ext", "redox_users", - "winapi", + "windows-sys 0.61.2", ] [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "dns-lookup" -version = "3.0.1" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e39034cee21a2f5bbb66ba0e3689819c4bb5d00382a282006e802a7ffa6c41d" +checksum = "6120e7e5dcd1c90098e349def389e2797377cdbe6b465ad8fdd115c0c99d5cf2" dependencies = [ "cfg-if", "libc", "socket2", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] -name = "dunce" -version = "1.0.5" +name = "drop_bomb" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1" [[package]] -name = "dyn-clone" -version = "1.0.20" +name = "dunce" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - -[[package]] -name = "embedded-io" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" - -[[package]] -name = "embedded-io" -version = "0.6.1" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "encode_unicode" @@ -1136,31 +1198,25 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "endian-type" -version = "0.1.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +checksum = "869b0adbda23651a9c5c0c3d270aac9fcb52e8622a8f2b17e57802d7791962f2" [[package]] name = "env_filter" -version = "0.1.4" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ "log", "regex", ] -[[package]] -name = "env_home" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f84e12ccf0a7ddc17a6c41c93326024c42920d7ee630d04950e6926645c0fe" - [[package]] name = "env_logger" -version = "0.11.8" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" dependencies = [ "anstream", "anstyle", @@ -1197,40 +1253,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - [[package]] name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - -[[package]] -name = "fd-lock" -version = "4.0.4" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" -dependencies = [ - "cfg-if", - "rustix", - "windows-sys 0.59.0", -] +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "find-msvc-tools" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" - -[[package]] -name = "flagset" -version = "0.4.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flame" @@ -1253,7 +1286,7 @@ checksum = "7693d9dd1ec1c54f52195dfe255b627f7cec7da33b679cd56de949e662b3db10" dependencies = [ "flame", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1269,21 +1302,10 @@ dependencies = [ ] [[package]] -name = "flate2" -version = "1.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" -dependencies = [ - "crc32fast", - "miniz_oxide", - "zlib-rs", -] - -[[package]] -name = "foldhash" -version = "0.1.5" +name = "fnv" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" @@ -1314,61 +1336,51 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", "futures-task", "pin-project-lite", - "pin-utils", "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "get-size-derive2" -version = "0.7.4" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b6d1e2f75c16bfbcd0f95d84f99858a6e2f885c2287d1f5c3a96e8444a34b4" +checksum = "c736d226c32e496b8377813b52269e11ad3a48d8373b68862d0364f04fd1229d" dependencies = [ "attribute-derive", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "get-size2" -version = "0.7.4" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49cf31a6d70300cf81461098f7797571362387ef4bf85d32ac47eaa59b3a5a1a" +checksum = "b411f34418305908ab15a82ff78958c2a9aee9a272b2a2e663836b20a4e4b9d3" dependencies = [ "compact_str", "get-size-derive2", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "ordermap", "smallvec", + "thin-vec", ] [[package]] @@ -1408,20 +1420,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" +dependencies = [ + "polyval", +] + [[package]] name = "gimli" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" dependencies = [ - "fallible-iterator", + "fnv", + "hashbrown 0.16.1", "indexmap", "stable_deref_trait", ] @@ -1432,6 +1466,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "graviola" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8596c4fa98466aae2fcf4c72a665bc0e021c0aaab1e47d82044d3dc3e309a76" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", +] + [[package]] name = "half" version = "2.7.1" @@ -1445,21 +1489,20 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "foldhash 0.1.5", - "serde", + "foldhash", ] [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ - "foldhash 0.2.0", + "foldhash", ] [[package]] @@ -1488,9 +1531,9 @@ checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" [[package]] name = "hmac" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ "digest", ] @@ -1505,65 +1548,171 @@ dependencies = [ ] [[package]] -name = "iana-time-zone" -version = "0.1.64" +name = "hybrid-array" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", + "typenum", ] [[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" +name = "icu_casemap" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +checksum = "070f98b5b82798fcb93654bf96ed9f40064fc44c86f51a09ea711092cd5cc5be" dependencies = [ - "cc", + "icu_casemap_data", + "icu_collections", + "icu_locale_core", + "icu_properties", + "icu_provider", + "potential_utf", + "writeable", + "zerovec", ] [[package]] -name = "indexmap" -version = "2.13.0" +name = "icu_casemap_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "846b0857ca091204be3c874bc93daaf89d4777e8d2d20b0d3ffe8f671d98014b" + +[[package]] +name = "icu_collections" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ - "equivalent", - "hashbrown 0.16.1", + "displaydoc", + "potential_utf", "serde", - "serde_core", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "indoc" -version = "2.0.7" +name = "icu_locale" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +checksum = "d5a396343c7208121dc86e35623d3dfe19814a7613cfd14964994cdc9c9a2e26" dependencies = [ - "rustversion", + "icu_collections", + "icu_locale_core", + "icu_locale_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "serde", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locale_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fdcc9ac77c6d74ff5cf6e65ef3181d6af32003b16fce3a77fb451d2f695993" + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "serde", + "stable_deref_trait", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", ] [[package]] name = "inout" -version = "0.1.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ "block-padding", - "generic-array", + "hybrid-array", ] [[package]] name = "insta" -version = "1.46.1" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248b42847813a1550dafd15296fd9748c651d0c32194559dbc05d804d54b21e8" +checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82" dependencies = [ "console", "once_cell", @@ -1579,14 +1728,25 @@ checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" [[package]] name = "is-macro" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d57a3e447e24c22647738e4607f1df1e0ec6f72e16182c4cd199f647cdfb0e4" +checksum = "8267aa6001e25494f3015f9663bbd88a18240c74483afa5f0934a1b3e4c388e9" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", ] [[package]] @@ -1595,6 +1755,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -1613,83 +1782,102 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.18" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ + "defmt", + "jiff-core", "jiff-static", + "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", ] [[package]] name = "jiff-static" -version = "0.2.18" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" dependencies = [ + "jiff-core", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] -name = "jni" -version = "0.21.1" +name = "jiff-tzdb" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" -dependencies = [ - "cesu8", - "cfg-if", - "combine", - "jni-sys", - "log", - "thiserror 1.0.69", - "walkdir", - "windows-sys 0.45.0", -] +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" [[package]] -name = "jni-sys" -version = "0.3.0" +name = "jiff-tzdb-platform" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] [[package]] name = "js-sys" -version = "0.3.85" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ - "once_cell", + "cfg-if", + "futures-util", "wasm-bindgen", ] [[package]] name = "junction" -version = "1.4.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "642883fdc81cf2da15ee8183fa1d2c7da452414dd41541a0f3e1428069345447" +checksum = "160f2eade097f30263b548aae5deb12ad349c909baa710fa24b92c9090b2e006" dependencies = [ "scopeguard", "windows-sys 0.61.2", @@ -1697,10 +1885,11 @@ dependencies = [ [[package]] name = "keccak" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" dependencies = [ + "cfg-if", "cpufeatures", ] @@ -1743,27 +1932,27 @@ checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] name = "lexopt" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa0e2a1fcbe2f6be6c42e342259976206b383122fc152e872795338b5a3f3a7" +checksum = "803ec87c9cfb29b9d2633f20cba1f488db3fd53f2158b1024cbefb47ba05d413" [[package]] name = "libbz2-rs-sys" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" -version = "0.2.180" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libffi" -version = "5.1.0" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0498fe5655f857803e156523e644dcdcdc3b3c7edda42ea2afdae2e09b2db87b" +checksum = "7a8d526bc07aa02e6826944ec446ecd8d228bd97b1cbd21e5bc5e3407a09a585" dependencies = [ "libc", "libffi-sys", @@ -1771,9 +1960,9 @@ dependencies = [ [[package]] name = "libffi-sys" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d4f1d4ce15091955144350b75db16a96d4a63728500122706fb4d29a26afbb" +checksum = "25831b230b6a90bdea9f28339c1d00d59773a1c492e8ca09b1ad80e56394c261" dependencies = [ "cc", ] @@ -1806,19 +1995,18 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.12" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" dependencies = [ - "bitflags 2.10.0", "libc", ] [[package]] name = "libsqlite3-sys" -version = "0.36.0" +version = "0.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b4103cffefa72eb8428cb6b47d6627161e51c2739fc5e3b734584157bc642a" +checksum = "f6c19a05435c21ac299d71b6a9c13db3e3f47c520517d58990a462a1397a61db" dependencies = [ "cc", "pkg-config", @@ -1827,18 +2015,24 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.5" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" +checksum = "03dcace986b149f29509af6ca70e6182bccce916b644424ecf484faa8ddc899a" dependencies = [ "zlib-rs", ] [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -1851,30 +2045,19 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lz4_flex" -version = "0.12.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "mac_address" version = "1.1.8" @@ -1896,9 +2079,9 @@ dependencies = [ [[package]] name = "malachite-base" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb2a098b227df48779e28ed4125dd3161b792a9254961377cea6f5c19e5b417" +checksum = "17073d2b5f3fe81b6abec0efcbdb6933d7adbda942854095f87c2f82633eafa5" dependencies = [ "hashbrown 0.16.1", "itertools 0.14.0", @@ -1908,9 +2091,9 @@ dependencies = [ [[package]] name = "malachite-bigint" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eaf19f1b8ba023528050372eafd72ca11f80f70c9dc2af9bb22f888bf079013" +checksum = "69c389baa355653795601ac65189e4ab21a1879fc1326a0244227f7757240edf" dependencies = [ "malachite-base", "malachite-nz", @@ -1921,9 +2104,9 @@ dependencies = [ [[package]] name = "malachite-nz" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7c0ddc4e2681459d70591baf30ca5abd31c25969e76d3605838bec794c8077" +checksum = "c2f37fc9ab5654d216d8ae22b57f0b8d9f1741ef160649d37ce1864a1e308993" dependencies = [ "itertools 0.14.0", "libm", @@ -1933,11 +2116,12 @@ dependencies = [ [[package]] name = "malachite-q" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13d19f04fc672f251d477c8d58c13c6c5550553dfba6a665c9ad3604466ac9a" +checksum = "6542042c11f3d94433ed4262cf5e82eb43eff687fc5bf1fe1b4b5cde2215836e" dependencies = [ "itertools 0.14.0", + "libm", "malachite-base", "malachite-nz", ] @@ -1951,7 +2135,7 @@ dependencies = [ "manyhow-macros", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1965,23 +2149,11 @@ dependencies = [ "quote", ] -[[package]] -name = "maplit" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" - -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "md-5" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" dependencies = [ "cfg-if", "digest", @@ -1989,15 +2161,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmap2" -version = "0.9.9" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" dependencies = [ "libc", ] @@ -2017,23 +2189,13 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", - "simd-adler32", -] - [[package]] name = "mt19937" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56bc7ea7924ea1a79a9e817d0483e39295424cf2b1276cf2b968f9a6c9b63b54" +checksum = "25b4ab1a6f4b7820876af86b84adf545d53a52f59c5374856225aad9562d903e" dependencies = [ - "rand_core 0.9.5", + "rand_core 0.10.1", ] [[package]] @@ -2051,7 +2213,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -2060,11 +2222,11 @@ dependencies = [ [[package]] name = "nix" -version = "0.30.1" +version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -2083,9 +2245,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -2102,15 +2264,15 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -2136,9 +2298,9 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" dependencies = [ "num_enum_derive", "rustversion", @@ -2146,13 +2308,13 @@ dependencies = [ [[package]] name = "num_enum_derive" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2175,9 +2337,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -2193,15 +2355,14 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "cfg-if", "foreign-types", "libc", - "once_cell", "openssl-macros", "openssl-sys", ] @@ -2214,7 +2375,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2225,18 +2386,18 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-src" -version = "300.5.4+3.5.4" +version = "300.6.1+3.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507b3792995dae9b0df8a1c1e3771e8418b7c2d9f0baeba32e6fe8b06c7cb72" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -2245,6 +2406,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "optional" version = "0.5.0" @@ -2253,23 +2420,13 @@ checksum = "978aa494585d3ca4ad74929863093e87cac9790d81fe7aba2b3dc2890643a0fc" [[package]] name = "ordermap" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfa78c92071bbd3628c22b1a964f7e0eb201dc1456555db072beb1662ecd6715" +checksum = "7f7476a5b122ff1fce7208e7ee9dccd0a516e835f5b8b19b8f3c98a34cf757c1" dependencies = [ "indexmap", ] -[[package]] -name = "page_size" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "parking_lot" version = "0.12.5" @@ -2283,8 +2440,7 @@ dependencies = [ [[package]] name = "parking_lot_core" version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +source = "git+https://github.com/youknowone/parking_lot?branch=rustpython#f4ee53a7b803354a8f0a6de2f28e93fc141240bf" dependencies = [ "cfg-if", "libc", @@ -2301,23 +2457,14 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pbkdf2" -version = "0.12.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ "digest", "hmac", ] -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -2338,12 +2485,12 @@ dependencies = [ [[package]] name = "phf" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" dependencies = [ "phf_macros", - "phf_shared 0.13.1", + "phf_shared 0.14.0", "serde", ] @@ -2364,30 +2511,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared 0.11.3", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] name = "phf_generator" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" dependencies = [ "fastrand", - "phf_shared 0.13.1", + "phf_shared 0.14.0", ] [[package]] name = "phf_macros" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085" dependencies = [ - "phf_generator 0.13.1", - "phf_shared 0.13.1", + "phf_generator 0.14.0", + "phf_shared 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2401,35 +2548,31 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" dependencies = [ "siphasher", ] [[package]] name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkcs5" -version = "0.7.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca" dependencies = [ "aes", + "aes-gcm", "cbc", "der", "pbkdf2", + "rand_core 0.10.1", "scrypt", "sha2", "spki", @@ -2437,21 +2580,21 @@ dependencies = [ [[package]] name = "pkcs8" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ "der", "pkcs5", - "rand_core 0.6.4", + "rand_core 0.10.1", "spki", ] [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "plotters" @@ -2489,34 +2632,44 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", +] + +[[package]] +name = "polyval" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfc63250416fea14f5749b90725916a6c903f599d51cb635aa7a52bfd03eede" +dependencies = [ + "cpubits", + "cpufeatures", + "universal-hash", ] [[package]] name = "portable-atomic" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.4" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] [[package]] -name = "postcard" -version = "1.1.3" +name = "potential_utf" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ - "cobs", - "embedded-io 0.4.0", - "embedded-io 0.6.1", - "serde", + "serde_core", + "writeable", + "zerovec", ] [[package]] @@ -2541,7 +2694,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.119", ] [[package]] @@ -2557,18 +2710,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "psm" -version = "0.1.29" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa96cb91275ed31d6da3e983447320c4eb219ac180fa1679a0889ff32861e2d" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" dependencies = [ "ar_archive_writer", "cc", @@ -2576,9 +2729,9 @@ dependencies = [ [[package]] name = "pymath" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfb6723b732fc7f0b29a0ee7150c7f70f947bf467b8c3e82530b13589a78b4c" +checksum = "bc10e50b7a1f2cc3887e983721cb51fc7574be0066c84bff3ef9e5c096e8d6d5" dependencies = [ "libc", "libm", @@ -2590,35 +2743,32 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.27.2" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ - "indoc", "libc", - "memoffset", "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", - "unindent", ] [[package]] name = "pyo3-build-config" -version = "0.27.2" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.27.2" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", "pyo3-build-config", @@ -2626,34 +2776,33 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.27.2" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "pyo3-macros-backend" -version = "0.27.2" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "quote" -version = "1.0.44" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -2677,7 +2826,7 @@ dependencies = [ "proc-macro-utils", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2686,6 +2835,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radium" version = "1.1.1" @@ -2697,9 +2852,9 @@ dependencies = [ [[package]] name = "radix_trie" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +checksum = "3b4431027dcd37fc2a73ef740b5f233aa805897935b8bce0195e41bbf9a3289a" dependencies = [ "endian-type", "nibble_vec", @@ -2707,23 +2862,24 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", - "rand_chacha 0.3.1", + "rand_chacha", "rand_core 0.6.4", ] [[package]] name = "rand" -version = "0.9.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.5", + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -2736,16 +2892,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", -] - [[package]] name = "rand_core" version = "0.6.4" @@ -2757,18 +2903,24 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.5" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rapidhash" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" dependencies = [ - "getrandom 0.3.4", + "rustversion", ] [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -2796,29 +2948,49 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", ] [[package]] name = "redox_users" -version = "0.4.6" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 1.0.69", + "thiserror", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] name = "regalloc2" -version = "0.13.5" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08effbc1fa53aaebff69521a5c05640523fab037b34a4a2c109506bc938246fa" +checksum = "757712e8e61590d6d4f5d563483755538b5aa13467837a3b41cd9832509a7f85" dependencies = [ "allocator-api2", "bumpalo", - "hashbrown 0.15.5", + "hashbrown 0.17.1", "log", "rustc-hash", "smallvec", @@ -2826,9 +2998,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -2838,9 +3010,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -2849,9 +3021,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "region" @@ -2883,7 +3055,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2896,81 +3068,15 @@ dependencies = [ "cfg-if", "getrandom 0.2.17", "libc", - "untrusted 0.9.0", + "untrusted", "windows-sys 0.52.0", ] -[[package]] -name = "ruff_python_ast" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=8b2e7b36f246b990fe473a84eef25ff429e59ecf#8b2e7b36f246b990fe473a84eef25ff429e59ecf" -dependencies = [ - "aho-corasick", - "bitflags 2.10.0", - "compact_str", - "get-size2", - "is-macro", - "memchr", - "ruff_python_trivia", - "ruff_source_file", - "ruff_text_size", - "rustc-hash", - "thiserror 2.0.18", -] - -[[package]] -name = "ruff_python_parser" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=8b2e7b36f246b990fe473a84eef25ff429e59ecf#8b2e7b36f246b990fe473a84eef25ff429e59ecf" -dependencies = [ - "bitflags 2.10.0", - "bstr", - "compact_str", - "get-size2", - "memchr", - "ruff_python_ast", - "ruff_python_trivia", - "ruff_text_size", - "rustc-hash", - "static_assertions", - "unicode-ident", - "unicode-normalization", - "unicode_names2 1.3.0", -] - -[[package]] -name = "ruff_python_trivia" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=8b2e7b36f246b990fe473a84eef25ff429e59ecf#8b2e7b36f246b990fe473a84eef25ff429e59ecf" -dependencies = [ - "itertools 0.14.0", - "ruff_source_file", - "ruff_text_size", - "unicode-ident", -] - -[[package]] -name = "ruff_source_file" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=8b2e7b36f246b990fe473a84eef25ff429e59ecf#8b2e7b36f246b990fe473a84eef25ff429e59ecf" -dependencies = [ - "memchr", - "ruff_text_size", -] - -[[package]] -name = "ruff_text_size" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?rev=8b2e7b36f246b990fe473a84eef25ff429e59ecf#8b2e7b36f246b990fe473a84eef25ff429e59ecf" -dependencies = [ - "get-size2", -] - [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rusticata-macros" @@ -2983,11 +3089,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -2996,9 +3102,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.36" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", "once_cell", @@ -3008,11 +3114,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-graviola" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf5d0a370be690f7f1aa4e1b912fde3f5f9a5c53285062fd2d6ac1a52ab3e883" +dependencies = [ + "graviola", + "rustls", +] + [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -3031,59 +3147,31 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "zeroize", ] -[[package]] -name = "rustls-platform-verifier" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" -dependencies = [ - "core-foundation 0.10.1", - "core-foundation-sys", - "jni", - "log", - "once_cell", - "rustls", - "rustls-native-certs", - "rustls-platform-verifier-android", - "rustls-webpki", - "security-framework", - "security-framework-sys", - "webpki-root-certs", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls-platform-verifier-android" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" - [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted 0.9.0", + "untrusted", ] [[package]] name = "rustpython" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "cfg-if", - "criterion", - "dirs-next", + "codspeed-criterion-compat", + "dirs", "env_logger", "flame", "flamescope", @@ -3091,374 +3179,464 @@ dependencies = [ "libc", "log", "pyo3", - "ruff_python_parser", + "rustls", + "rustls-graviola", + "rustpython-capi", "rustpython-compiler", "rustpython-pylib", + "rustpython-ruff_python_parser", "rustpython-stdlib", "rustpython-vm", "rustyline", "winresource", ] +[[package]] +name = "rustpython-capi" +version = "0.5.0" +dependencies = [ + "bitflags 2.13.1", + "itertools 0.15.0", + "libc", + "malachite-bigint", + "num-complex", + "pyo3", + "rustpython-pylib", + "rustpython-stdlib", + "rustpython-vm", +] + [[package]] name = "rustpython-codegen" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "ahash", - "bitflags 2.10.0", + "bitflags 2.13.1", "indexmap", - "insta", - "itertools 0.14.0", + "itertools 0.15.0", "log", "malachite-bigint", "memchr", "num-complex", "num-traits", - "ruff_python_ast", - "ruff_python_parser", - "ruff_text_size", + "rapidhash", "rustpython-compiler-core", "rustpython-literal", + "rustpython-ruff_python_ast", + "rustpython-ruff_python_parser", + "rustpython-ruff_text_size", + "rustpython-unicode", "rustpython-wtf8", - "thiserror 2.0.18", - "unicode_names2 2.0.0", + "thiserror", ] [[package]] name = "rustpython-common" -version = "0.4.0" +version = "0.5.0" dependencies = [ "ascii", - "bitflags 2.10.0", - "cfg-if", - "getrandom 0.3.4", - "itertools 0.14.0", + "base64", + "bitflags 2.13.1", + "blake2b_simd", + "blake2s_simd", + "bzip2", + "crc32fast", + "getrandom 0.4.3", + "itertools 0.15.0", "libc", + "libz-rs-sys", "lock_api", "malachite-base", "malachite-bigint", "malachite-q", - "nix 0.30.1", + "md-5", + "memchr", "num-complex", "num-traits", - "once_cell", "parking_lot", "radium", "rustpython-literal", + "rustpython-unicode", "rustpython-wtf8", + "scrypt", + "sha1", + "sha2", + "sha3", + "shake", "siphasher", - "unicode_names2 2.0.0", - "widestring", - "windows-sys 0.61.2", + "xz", + "xz-sys", + "zlib-rs", ] [[package]] name = "rustpython-compiler" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "ruff_python_ast", - "ruff_python_parser", - "ruff_source_file", - "ruff_text_size", "rustpython-codegen", "rustpython-compiler-core", - "thiserror 2.0.18", + "rustpython-ruff_python_ast", + "rustpython-ruff_python_parser", + "rustpython-ruff_source_file", + "rustpython-ruff_text_size", + "thiserror", ] [[package]] name = "rustpython-compiler-core" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "bitflags 2.10.0", - "itertools 0.14.0", + "bitflags 2.13.1", + "bitflagset", + "itertools 0.15.0", "lz4_flex", "malachite-bigint", "num-complex", - "ruff_source_file", + "num-traits", + "rustpython-ruff_source_file", "rustpython-wtf8", ] -[[package]] -name = "rustpython-compiler-source" -version = "0.5.0+deprecated" -dependencies = [ - "ruff_source_file", - "ruff_text_size", -] - [[package]] name = "rustpython-derive" -version = "0.4.0" +version = "0.5.0" dependencies = [ "rustpython-compiler", "rustpython-derive-impl", - "syn", + "syn 2.0.119", ] [[package]] name = "rustpython-derive-impl" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "itertools 0.14.0", - "maplit", + "itertools 0.15.0", "proc-macro2", "quote", "rustpython-compiler-core", "rustpython-doc", - "syn", + "syn 2.0.119", "syn-ext", "textwrap", ] [[package]] name = "rustpython-doc" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "phf 0.13.1", + "phf 0.14.0", ] [[package]] -name = "rustpython-jit" -version = "0.4.0" +name = "rustpython-host_env" +version = "0.5.0" dependencies = [ - "approx", - "cranelift", - "cranelift-jit", - "cranelift-module", + "bitflags 2.13.1", + "cc", + "der", + "dns-lookup", + "gethostname", + "getrandom 0.4.3", + "junction", + "libc", "libffi", + "libloading 0.9.0", + "mac_address", + "memchr", + "memmap2", + "nix 0.31.3", "num-traits", - "rustpython-compiler-core", - "rustpython-derive", - "rustpython-wtf8", - "thiserror 2.0.18", -] - + "num_cpus", + "parking_lot", + "paste", + "pkcs8", + "rustix", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-webpki", + "rustpython-wtf8", + "rustyline", + "schannel", + "sha2", + "socket2", + "system-configuration", + "termios", + "which", + "widestring", + "windows-sys 0.61.2", + "x509-parser", +] + +[[package]] +name = "rustpython-jit" +version = "0.5.0" +dependencies = [ + "approx", + "cranelift", + "cranelift-jit", + "cranelift-module", + "libffi", + "num-traits", + "rustpython-compiler-core", + "rustpython-derive", + "rustpython-wtf8", + "thiserror", +] + [[package]] name = "rustpython-literal" -version = "0.4.0" +version = "0.5.0" dependencies = [ "hexf-parse", "is-macro", "lexical-parse-float", "num-traits", - "rand 0.9.2", + "rand 0.10.2", + "rustpython-unicode", "rustpython-wtf8", - "unic-ucd-category", ] [[package]] name = "rustpython-pylib" -version = "0.4.0" +version = "0.5.0" dependencies = [ "glob", "rustpython-compiler-core", "rustpython-derive", ] +[[package]] +name = "rustpython-ruff_python_ast" +version = "0.16.5" +source = "git+https://github.com/RustPython/ruff.git?tag=0.16.5-rustpython#df59fe4c720b48a29fe8726a37e2ef06322096f8" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags 2.13.1", + "char_str", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "rustc-hash", + "rustpython-ruff_python_trivia", + "rustpython-ruff_source_file", + "rustpython-ruff_text_size", + "thin-vec", + "thiserror", +] + +[[package]] +name = "rustpython-ruff_python_parser" +version = "0.16.5" +source = "git+https://github.com/RustPython/ruff.git?tag=0.16.5-rustpython#df59fe4c720b48a29fe8726a37e2ef06322096f8" +dependencies = [ + "bitflags 2.13.1", + "bstr", + "drop_bomb", + "get-size2", + "memchr", + "rustc-hash", + "rustpython-ruff_python_ast", + "rustpython-ruff_python_trivia", + "rustpython-ruff_text_size", + "stacker", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", + "unicode_names2 1.3.0", +] + +[[package]] +name = "rustpython-ruff_python_trivia" +version = "0.16.5" +source = "git+https://github.com/RustPython/ruff.git?tag=0.16.5-rustpython#df59fe4c720b48a29fe8726a37e2ef06322096f8" +dependencies = [ + "itertools 0.15.0", + "rustc-hash", + "rustpython-ruff_source_file", + "rustpython-ruff_text_size", + "unicode-ident", +] + +[[package]] +name = "rustpython-ruff_source_file" +version = "0.16.5" +source = "git+https://github.com/RustPython/ruff.git?tag=0.16.5-rustpython#df59fe4c720b48a29fe8726a37e2ef06322096f8" +dependencies = [ + "memchr", + "rustpython-ruff_text_size", +] + +[[package]] +name = "rustpython-ruff_text_size" +version = "0.16.5" +source = "git+https://github.com/RustPython/ruff.git?tag=0.16.5-rustpython#df59fe4c720b48a29fe8726a37e2ef06322096f8" +dependencies = [ + "get-size2", +] + [[package]] name = "rustpython-sre_engine" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "bitflags 2.10.0", - "criterion", + "bitflags 2.13.1", + "codspeed-criterion-compat", "num_enum", "optional", + "rustpython-unicode", "rustpython-wtf8", ] [[package]] name = "rustpython-stdlib" -version = "0.4.0" +version = "0.5.0" dependencies = [ "adler32", - "ahash", "ascii", - "aws-lc-rs", - "base64", - "blake2", - "bzip2", - "cfg-if", - "chrono", - "crc32fast", + "constant_time_eq 0.6.0", "crossbeam-utils", "csv-core", - "der", - "digest", - "dns-lookup", - "dyn-clone", "flame", - "flate2", "foreign-types-shared", - "gethostname", "hex", "indexmap", - "itertools 0.14.0", + "insta", + "itertools 0.15.0", "libc", "libsqlite3-sys", - "libz-rs-sys", - "lzma-sys", - "mac_address", "malachite-bigint", - "md-5", "memchr", - "memmap2", "mt19937", - "nix 0.30.1", "num-complex", - "num-integer", "num-traits", "num_enum", - "oid-registry", "openssl", - "openssl-probe", "openssl-sys", - "page_size", "parking_lot", "paste", - "pem-rfc7468 1.0.0", - "phf 0.13.1", - "pkcs8", + "pem-rfc7468", + "phf 0.14.0", "pymath", - "rand_core 0.9.5", - "rustix", + "rand 0.10.2", + "rapidhash", "rustls", - "rustls-native-certs", "rustls-pemfile", - "rustls-platform-verifier", "rustpython-common", "rustpython-derive", + "rustpython-host_env", + "rustpython-pylib", + "rustpython-ruff_python_ast", + "rustpython-ruff_python_parser", + "rustpython-ruff_source_file", + "rustpython-ruff_text_size", + "rustpython-unicode", "rustpython-vm", - "schannel", - "sha-1", - "sha2", - "sha3", - "socket2", - "system-configuration", + "scopeguard", "tcl-sys", - "termios", "tk-sys", - "ucd", - "unic-char-property", - "unic-normal", - "unic-ucd-age", - "unic-ucd-bidi", - "unic-ucd-category", - "unic-ucd-ident", - "unicode-bidi-mirroring", - "unicode-casing", - "unicode_names2 2.0.0", "uuid", "webpki-roots", "widestring", - "windows-sys 0.61.2", - "x509-cert", "x509-parser", "xml", - "xz2", ] [[package]] -name = "rustpython-venvlauncher" -version = "0.4.0" +name = "rustpython-unicode" +version = "0.5.0" dependencies = [ - "windows-sys 0.61.2", + "icu_casemap", + "icu_locale", + "icu_normalizer", + "icu_properties", + "rustpython-wtf8", + "unicode_names2 3.1.0", + "writeable", ] +[[package]] +name = "rustpython-venvlauncher" +version = "0.5.0" + [[package]] name = "rustpython-vm" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "ahash", "ascii", - "bitflags 2.10.0", + "bitflags 2.13.1", "bstr", - "caseless", - "cfg-if", - "chrono", - "constant_time_eq", + "constant_time_eq 0.6.0", "crossbeam-utils", - "errno", "exitcode", "flame", "flamer", - "getrandom 0.3.4", "glob", "half", "hex", "indexmap", "is-macro", - "itertools 0.14.0", - "junction", + "itertools 0.15.0", + "itoa", + "jiff", "libc", - "libffi", - "libloading 0.9.0", "log", "malachite-bigint", "memchr", - "nix 0.30.1", "num-complex", "num-integer", "num-traits", - "num_cpus", "num_enum", - "once_cell", "optional", "parking_lot", "paste", "psm", + "rapidhash", "result-like", - "ruff_python_ast", - "ruff_python_parser", - "ruff_text_size", - "rustix", "rustpython-codegen", "rustpython-common", "rustpython-compiler", "rustpython-compiler-core", "rustpython-derive", + "rustpython-host_env", "rustpython-jit", "rustpython-literal", + "rustpython-ruff_python_ast", + "rustpython-ruff_python_parser", + "rustpython-ruff_text_size", "rustpython-sre_engine", - "rustyline", - "scoped-tls", + "rustpython-unicode", "scopeguard", "serde_core", "static_assertions", "strum", "strum_macros", - "thiserror 2.0.18", - "thread_local", - "timsort", - "uname", - "unic-ucd-bidi", - "unic-ucd-category", - "unic-ucd-ident", - "unicode-casing", + "thin-vec", + "thiserror", "wasm-bindgen", - "which", "widestring", - "windows-sys 0.61.2", ] [[package]] name = "rustpython-wtf8" -version = "0.4.0" +version = "0.5.0" dependencies = [ "ascii", "bstr", - "itertools 0.14.0", + "itertools 0.15.0", "memchr", ] [[package]] name = "rustpython_wasm" -version = "0.4.0" +version = "0.5.0" dependencies = [ "console_error_panic_hook", "js-sys", - "ruff_python_parser", "rustpython-common", "rustpython-pylib", + "rustpython-ruff_text_size", "rustpython-stdlib", "rustpython-vm", "serde-wasm-bindgen", @@ -3470,53 +3648,53 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "rustyline" -version = "17.0.2" +version = "18.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" +checksum = "53f6a737db68eb1a8ccff86b584b2fc13eca6a7bb6f78ebc7c529547e3ab9684" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.13.1", "cfg-if", "clipboard-win", - "fd-lock", "home", "libc", "log", "memchr", - "nix 0.30.1", + "nix 0.31.3", "radix_trie", "unicode-segmentation", "unicode-width", "utf8parse", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "ryu" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "safe_arch" -version = "0.9.3" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629516c85c29fe757770fa03f2074cf1eac43d44c02a3de9fc2ef7b0e207dfdd" +checksum = "1f7caad094bd561859bcd467734a720c3c1f5d1f338995351fefe2190c45efed" dependencies = [ "bytemuck", ] [[package]] name = "salsa20" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +checksum = "2f874456e72520ff1375a06c588eaf074b0f01f9e9e1aada45bd9b7954a6e42c" dependencies = [ + "cfg-if", "cipher", ] @@ -3531,19 +3709,13 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "scopeguard" version = "1.2.0" @@ -3552,10 +3724,11 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "scrypt" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +checksum = "d87af57419b594aa23fa95f09f0e06d80d84ba01c26148c43844cad6ff4485f0" dependencies = [ + "cfg-if", "pbkdf2", "salsa20", "sha2", @@ -3563,12 +3736,12 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.5.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.10.0", - "core-foundation 0.10.1", + "bitflags 2.13.1", + "core-foundation", "core-foundation-sys", "libc", "security-framework-sys", @@ -3576,9 +3749,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.15.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -3622,14 +3795,14 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -3640,18 +3813,18 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.0.4" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ "serde_core", ] [[package]] -name = "sha-1" -version = "0.10.1" +name = "sha1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", "cpufeatures", @@ -3659,10 +3832,10 @@ dependencies = [ ] [[package]] -name = "sha1" -version = "0.10.6" +name = "sha2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", "cpufeatures", @@ -3670,24 +3843,25 @@ dependencies = [ ] [[package]] -name = "sha2" -version = "0.10.9" +name = "sha3" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "bc9bad02c26382724b2d2692c6f179285e4b54eeecd7968f52a50059c3c11759" dependencies = [ - "cfg-if", - "cpufeatures", "digest", + "keccak", + "sponge-cursor", ] [[package]] -name = "sha3" -version = "0.10.8" +name = "shake" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +checksum = "09057cb2149ad4cbd2da1e26b351f9a4c354219421229c69c3063e6f61947c4a" dependencies = [ "digest", "keccak", + "sponge-cursor", ] [[package]] @@ -3705,19 +3879,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "rand_core 0.6.4", -] - -[[package]] -name = "simd-adler32" -version = "0.3.8" +name = "shlex" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "similar" @@ -3727,73 +3892,99 @@ checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "siphasher" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "slab" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -dependencies = [ - "serde", -] +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "spki" -version = "0.7.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" dependencies = [ "base64ct", "der", ] +[[package]] +name = "sponge-cursor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "strum" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" [[package]] name = "strum_macros" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3804,9 +3995,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.114" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -3821,7 +4023,7 @@ checksum = "b126de4ef6c2a628a68609dd00733766c3b015894698a438ebdf374933fc31d1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3832,17 +4034,17 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "system-configuration" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +checksum = "501336eb7ba9e417300a6a0fa985721065467aa83a6dcf0422a8e43e4c0328fa" dependencies = [ - "bitflags 2.10.0", - "core-foundation 0.9.4", + "bitflags 2.13.1", + "core-foundation", "system-configuration-sys", ] @@ -3858,9 +4060,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "tcl-sys" @@ -3873,12 +4075,12 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.24.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -3900,43 +4102,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" [[package]] -name = "thiserror" -version = "1.0.69" +name = "thin-vec" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] +checksum = "79def32ffcd477db1ff26f76dab9e3a91f0bd42a85ca96577089b24623056f9d" [[package]] name = "thiserror" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" -dependencies = [ - "thiserror-impl 2.0.18", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "proc-macro2", - "quote", - "syn", + "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -3950,23 +4138,13 @@ dependencies = [ "winapi", ] -[[package]] -name = "thread_local" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" -dependencies = [ - "cfg-if", -] - [[package]] name = "time" -version = "0.3.46" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -3976,25 +4154,30 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.26" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", ] [[package]] -name = "timsort" -version = "0.1.3" +name = "tinystr" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "639ce8ef6d2ba56be0383a94dd13b92138d58de44c62618303bb798fa92bdc00" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "serde_core", + "zerovec", +] [[package]] name = "tinytemplate" @@ -4008,9 +4191,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -4031,31 +4214,10 @@ dependencies = [ ] [[package]] -name = "tls_codec" -version = "0.4.2" +name = "toml" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" -dependencies = [ - "tls_codec_derive", - "zeroize", -] - -[[package]] -name = "tls_codec_derive" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "toml" -version = "0.9.11+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3afc9a848309fe1aaffaed6e1546a7a14de1f935dc9d89d32afd9a44bab7c46" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ "indexmap", "serde_core", @@ -4068,27 +4230,27 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.5+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] [[package]] name = "toml_parser" -version = "1.0.6+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.0.6+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" [[package]] name = "twox-hash" @@ -4098,147 +4260,15 @@ checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "typenum" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" - -[[package]] -name = "ucd" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4fa6e588762366f1eb4991ce59ad1b93651d0b769dfb4e4d1c5c4b943d1159" - -[[package]] -name = "uname" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8" -dependencies = [ - "libc", -] - -[[package]] -name = "unic-char-property" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" -dependencies = [ - "unic-char-range", -] - -[[package]] -name = "unic-char-range" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" - -[[package]] -name = "unic-common" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" - -[[package]] -name = "unic-normal" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09d64d33589a94628bc2aeb037f35c2e25f3f049c7348b5aa5580b48e6bba62" -dependencies = [ - "unic-ucd-normal", -] - -[[package]] -name = "unic-ucd-age" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8cfdfe71af46b871dc6af2c24fcd360e2f3392ee4c5111877f2947f311671c" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-bidi" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d568b51222484e1f8209ce48caa6b430bf352962b877d592c29ab31fb53d8c" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-category" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8d4591f5fcfe1bd4453baaf803c40e1b1e69ff8455c47620440b46efef91c0" -dependencies = [ - "matches", - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-hangul" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1dc690e19010e1523edb9713224cba5ef55b54894fe33424439ec9a40c0054" -dependencies = [ - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-ident" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-normal" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86aed873b8202d22b13859dda5fe7c001d271412c31d411fd9b827e030569410" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-hangul", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-version" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" -dependencies = [ - "unic-common", -] - -[[package]] -name = "unicode-bidi-mirroring" -version = "0.4.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfa6e8c60bb66d49db113e0125ee8711b7647b5579dc7f5f19c42357ed039fe" - -[[package]] -name = "unicode-casing" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061dbb8cc7f108532b6087a0065eff575e892a4bcb503dc57323a197457cc202" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-normalization" @@ -4251,9 +4281,9 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-width" @@ -4273,12 +4303,12 @@ dependencies = [ [[package]] name = "unicode_names2" -version = "2.0.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d189085656ca1203291e965444e7f6a2723fbdd1dd9f34f8482e79bafd8338a0" +checksum = "82c3e18d850bb6ebd57735e5654f0af65e572b05c2397e7b2b1a7c6a792cc29c" dependencies = [ "phf 0.11.3", - "unicode_names2_generator 2.0.0", + "unicode_names2_generator 3.1.0", ] [[package]] @@ -4290,36 +4320,46 @@ dependencies = [ "getopts", "log", "phf_codegen", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] name = "unicode_names2_generator" -version = "2.0.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1262662dc96937c71115228ce2e1d30f41db71a7a45d3459e98783ef94052214" +checksum = "849744a58c479122ff24910d7ca57f312b62613ef0412dc327776ae6b235d16a" dependencies = [ "phf_codegen", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] -name = "unindent" -version = "0.2.4" +name = "universal-hash" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common", + "ctutils", +] [[package]] name = "untrusted" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] -name = "untrusted" -version = "0.9.0" +name = "utf16_iter" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "utf8parse" @@ -4329,9 +4369,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.20.0" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "atomic", "js-sys", @@ -4368,18 +4408,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.108" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -4390,23 +4430,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.58" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.108" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4414,124 +4450,80 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.108" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.108" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] [[package]] -name = "wasmparser" -version = "0.243.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6d8db401b0528ec316dfbe579e6ab4152d61739cfe076706d2009127970159d" -dependencies = [ - "bitflags 2.10.0", - "hashbrown 0.15.5", - "indexmap", - "serde", -] - -[[package]] -name = "wasmtime-environ" -version = "41.0.1" +name = "wasmtime-internal-core" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b9af430b11ff3cd63fbef54cf38e26154089c179316b8a5e400b8ba2d0ebf1" +checksum = "821385794cf44baf2967fd2515368429ed2bff9b9b256f62d09a0f1301474fd8" dependencies = [ - "anyhow", - "cranelift-bitset", - "cranelift-entity", - "gimli", - "indexmap", - "log", - "object", - "postcard", - "serde", - "serde_derive", - "smallvec", - "target-lexicon", - "wasmparser", + "hashbrown 0.17.1", + "libm", ] [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "41.0.1" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b46da671c07242b5f5eab491b12d6c25dd26929f1693c055fcca94489ef8f5" +checksum = "3cce25afd9e7ac4957292a5c6a219951dd0f140b8417e7378a97f50b67c3b96f" dependencies = [ - "cfg-if", "libc", - "wasmtime-environ", + "wasmtime-internal-core", "windows-sys 0.61.2", ] -[[package]] -name = "wasmtime-internal-math" -version = "41.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d1f0763c6f6f78e410f964db9f53d9b84ab4cc336945e81f0b78717b0a9934e" -dependencies = [ - "libm", -] - [[package]] name = "web-sys" -version = "0.3.85" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", ] -[[package]] -name = "webpki-root-certs" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36a29fc0408b113f68cf32637857ab740edfafdf460c326cd2afaa2d84cc05dc" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "webpki-roots" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" dependencies = [ "rustls-pki-types", ] [[package]] name = "which" -version = "8.0.0" +version = "8.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fabb953106c3c8eea8306e4393700d7657561cb43122571b172bbfb7c7ba1d" +checksum = "8f3ef584124b911bcc3875c2f1472e80f24361ceb789bd1c62b3e9a3df9ff43c" dependencies = [ - "env_home", - "rustix", - "winsafe", + "libc", ] [[package]] name = "wide" -version = "0.8.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13ca908d26e4786149c48efcf6c0ea09ab0e06d1fe3c17dc1b4b0f1ca4a7e788" +checksum = "dfdfe6a32973f2d1b268b8895845a8a96cac2f0191e72c27cc929036060dbf89" dependencies = [ "bytemuck", "safe_arch", @@ -4574,99 +4566,19 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -4678,239 +4590,109 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" -version = "0.7.14" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" [[package]] name = "winresource" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e287ced0f21cd11f4035fe946fd3af145f068d1acb708afd248100f89ec7432d" +checksum = "0986a8b1d586b7d3e4fe3d9ea39fb451ae22869dcea4aa109d287a374d866087" dependencies = [ "toml", "version_check", ] [[package]] -name = "winsafe" -version = "0.0.19" +name = "wit-bindgen" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] -name = "wit-bindgen" -version = "0.51.0" +name = "write16" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" [[package]] -name = "x509-cert" -version = "0.2.5" +name = "writeable" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" -dependencies = [ - "const-oid", - "der", - "sha1", - "signature", - "spki", - "tls_codec", -] +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "x509-parser" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3e137310115a65136898d2079f003ce33331a6c4b0d51f1531d1be082b6425" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" dependencies = [ "asn1-rs", "data-encoding", @@ -4919,73 +4701,159 @@ dependencies = [ "nom", "oid-registry", "rusticata-macros", - "thiserror 2.0.18", + "thiserror", "time", ] [[package]] name = "xml" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] -name = "xz2" -version = "0.1.7" +name = "xz" +version = "0.4.6-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7af8851c229f574c5e4f4e9b6d90ca06c43974a563f7f42de4b88d651bf8f19c" +dependencies = [ + "xz-core", +] + +[[package]] +name = "xz-core" +version = "0.1.0-rc.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +checksum = "ef4d12cf9c122b2523cce22d6ab3083f4eb56d1221e5d30ddcc70fbaac553589" dependencies = [ - "lzma-sys", + "libc", + "memchr", + "windows-sys 0.61.2", +] + +[[package]] +name = "xz-sys" +version = "0.4.6-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "114e3a14e5bb2d46513305741650595041b7e7e1677aa5c9715a09b7a8da08fd" +dependencies = [ + "libc", + "xz-core", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.34" +version = "0.8.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71ddd76bcebeed25db614f82bf31a9f4222d3fbba300e6fb6c00afa26cbd4d9d" +checksum = "75726053136156d419e285b9b7eddaaea9e3fea6ce32eed44a89901f0bd98de1" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.34" +version = "0.8.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8187381b52e32220d50b255276aa16a084ec0a9017a0ca2152a1f55c539758d" +checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ - "zeroize_derive", + "serde", + "yoke", + "zerofrom", + "zerovec-derive", ] [[package]] -name = "zeroize_derive" -version = "1.4.3" +name = "zerovec-derive" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "zlib-rs" -version = "0.5.5" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" [[package]] name = "zmij" -version = "1.0.17" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02aae0f83f69aafc94776e879363e9771d7ecbffe2c7fbb6c14c5e00dfe88439" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml index 54d1fdda41f..49afd499f5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,9 @@ repository.workspace = true license.workspace = true [features] -default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls"] +capi = ["dep:rustpython-capi", "threading"] +default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls-aws-lc", "host_env"] +host_env = ["rustpython-vm/host_env", "rustpython-stdlib?/host_env"] importlib = ["rustpython-vm/importlib"] encodings = ["rustpython-vm/encodings"] stdio = ["rustpython-vm/stdio"] @@ -20,31 +22,34 @@ freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/fre jit = ["rustpython-vm/jit"] threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"] sqlite = ["rustpython-stdlib/sqlite"] -ssl = [] +ssl = ["host_env"] ssl-rustls = ["ssl", "rustpython-stdlib/ssl-rustls"] +ssl-rustls-aws-lc = ["ssl-rustls", "dep:rustls", "rustls/aws_lc_rs"] +ssl-rustls-aws-lc-fips = ["ssl-rustls-aws-lc", "rustls/fips"] ssl-openssl = ["ssl", "rustpython-stdlib/ssl-openssl"] -ssl-vendor = ["ssl-openssl", "rustpython-stdlib/ssl-vendor"] +ssl-openssl-vendor = ["ssl-openssl", "rustpython-stdlib/ssl-openssl-vendor"] tkinter = ["rustpython-stdlib/tkinter"] [build-dependencies] winresource = "0.1" [dependencies] +rustpython-capi = { workspace = true, optional = true } rustpython-compiler = { workspace = true } rustpython-pylib = { workspace = true, optional = true } rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] } -rustpython-vm = { workspace = true, features = ["compiler"] } -ruff_python_parser = { workspace = true } +rustpython-vm = { workspace = true, features = ["compiler", "gc"] } -cfg-if = { workspace = true } log = { workspace = true } flame = { workspace = true, optional = true } lexopt = "0.3" -dirs = { package = "dirs-next", version = "2.0" } +dirs = "6" env_logger = "0.11" flamescope = { version = "0.1.2", optional = true } +rustls = { workspace = true, optional = true } + [target.'cfg(windows)'.dependencies] libc = { workspace = true } @@ -53,8 +58,10 @@ rustyline = { workspace = true } [dev-dependencies] criterion = { workspace = true } -pyo3 = { version = "0.27", features = ["auto-initialize"] } +pyo3 = { workspace = true, features = ["auto-initialize"] } +rustls-graviola = { workspace = true } rustpython-stdlib = { workspace = true } +ruff_python_parser = { workspace = true } [[bench]] name = "execution" @@ -68,6 +75,16 @@ harness = false name = "rustpython" path = "src/main.rs" +[[example]] +name = "custom_tls_providers" +path = "examples/custom_tls_providers.rs" +required-features = [ + "rustls/ring", + "rustpython-pylib/freeze-stdlib", + "rustpython-stdlib/ssl-rustls", + "rustpython-vm/freeze-stdlib", +] + [profile.dev.package."*"] opt-level = 3 @@ -76,10 +93,21 @@ opt-level = 3 # https://github.com/rust-lang/rust/issues/92869 # lto = "thin" -# Doesn't change often +# Some crates don't change as much but benefit more from +# more expensive optimization passes, so we selectively +# decrease codegen-units in some cases. [profile.release.package.rustpython-doc] codegen-units = 1 +[profile.release.package.rustpython-literal] +codegen-units = 1 + +[profile.release.package.rustpython-common] +codegen-units = 1 + +[profile.release.package.rustpython-wtf8] +codegen-units = 1 + [profile.bench] lto = "thin" codegen-units = 1 @@ -88,7 +116,16 @@ opt-level = 3 [profile.release] lto = "thin" +[profile.wasm-release] +inherits = "release" +opt-level = "s" +lto = true +codegen-units = 1 +strip = true +panic = "abort" + [patch.crates-io] +parking_lot_core = { git = "https://github.com/youknowone/parking_lot", branch = "rustpython" } # REDOX START, Uncomment when you want to compile/check with redoxer # REDOX END @@ -120,101 +157,167 @@ members = [ exclude = ["pymath"] [workspace.package] -version = "0.4.0" +version = "0.5.0" authors = ["RustPython Team"] edition = "2024" -rust-version = "1.93.0" +rust-version = "1.95.0" repository = "https://github.com/RustPython/RustPython" license = "MIT" [workspace.dependencies] -rustpython-compiler-core = { path = "crates/compiler-core", version = "0.4.0" } -rustpython-compiler = { path = "crates/compiler", version = "0.4.0" } -rustpython-codegen = { path = "crates/codegen", version = "0.4.0" } -rustpython-common = { path = "crates/common", version = "0.4.0" } -rustpython-derive = { path = "crates/derive", version = "0.4.0" } -rustpython-derive-impl = { path = "crates/derive-impl", version = "0.4.0" } -rustpython-jit = { path = "crates/jit", version = "0.4.0" } -rustpython-literal = { path = "crates/literal", version = "0.4.0" } -rustpython-vm = { path = "crates/vm", default-features = false, version = "0.4.0" } -rustpython-pylib = { path = "crates/pylib", version = "0.4.0" } -rustpython-stdlib = { path = "crates/stdlib", default-features = false, version = "0.4.0" } -rustpython-sre_engine = { path = "crates/sre_engine", version = "0.4.0" } -rustpython-wtf8 = { path = "crates/wtf8", version = "0.4.0" } -rustpython-doc = { path = "crates/doc", version = "0.4.0" } - -# Ruff tag 0.14.14 is based on commit 8b2e7b36f246b990fe473a84eef25ff429e59ecf -# at the time of this capture. We use the commit hash to ensure reproducible builds. -ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", rev = "8b2e7b36f246b990fe473a84eef25ff429e59ecf" } -ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", rev = "8b2e7b36f246b990fe473a84eef25ff429e59ecf" } -ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", rev = "8b2e7b36f246b990fe473a84eef25ff429e59ecf" } -ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", rev = "8b2e7b36f246b990fe473a84eef25ff429e59ecf" } - -phf = { version = "0.13.1", default-features = false, features = ["macros"]} -ahash = "0.8.12" -ascii = "1.1" -bitflags = "2.9.4" -bstr = "1" -cfg-if = "1.0" -chrono = { version = "0.4.43", default-features = false, features = ["clock", "oldtime", "std"] } -constant_time_eq = "0.4" -criterion = { version = "0.8", features = ["html_reports"] } +rustpython-capi = { path = "crates/capi", version = "0.5.0" } +rustpython-compiler-core = { path = "crates/compiler-core", version = "0.5.0" } +rustpython-compiler = { path = "crates/compiler", version = "0.5.0" } +rustpython-codegen = { path = "crates/codegen", version = "0.5.0" } +rustpython-common = { path = "crates/common", version = "0.5.0" } +rustpython-host_env = { path = "crates/host_env", version = "0.5.0" } +rustpython-derive = { path = "crates/derive", version = "0.5.0" } +rustpython-derive-impl = { path = "crates/derive-impl", version = "0.5.0" } +rustpython-jit = { path = "crates/jit", version = "0.5.0" } +rustpython-literal = { path = "crates/literal", version = "0.5.0" } +rustpython-vm = { path = "crates/vm", default-features = false, version = "0.5.0" } +rustpython-pylib = { path = "crates/pylib", version = "0.5.0" } +rustpython-stdlib = { path = "crates/stdlib", default-features = false, version = "0.5.0" } +rustpython-sre_engine = { path = "crates/sre_engine", version = "0.5.0" } +rustpython-unicode = { path = "crates/unicode", version = "0.5.0" } +rustpython-wtf8 = { path = "crates/wtf8", version = "0.5.0" } +rustpython-doc = { path = "crates/doc", version = "0.5.0" } + +# Use the RustPython Ruff fork for RustPython public `_ast` metadata. +ruff_python_parser = { package = "rustpython-ruff_python_parser", git = "https://github.com/RustPython/ruff.git", tag = "0.16.5-rustpython" } +ruff_python_ast = { package = "rustpython-ruff_python_ast", git = "https://github.com/RustPython/ruff.git", tag = "0.16.5-rustpython" } +ruff_text_size = { package = "rustpython-ruff_text_size", git = "https://github.com/RustPython/ruff.git", tag = "0.16.5-rustpython" } +ruff_source_file = { package = "rustpython-ruff_source_file", git = "https://github.com/RustPython/ruff.git", tag = "0.16.5-rustpython" } + +der = { version = "0.8", features = ["alloc", "oid", "pem", "zeroize"] } +phf = { version = "0.14.0", default-features = false, features = ["macros"]} +adler32 = "1.2.0" +approx = "0.5.1" +ascii = { version = "1.1", default-features = false } +base64 = "0.23" +blake2b_simd = "1.0.4" +blake2s_simd = "1.0.4" +bitflags = "2.11.0" +bitflagset = "0.0.3" +bstr = { version = "1", default-features = false, features = ["unicode"] } +bzip2 = "0.6" +console_error_panic_hook = "0.1" +constant_time_eq = "0.6" +cranelift = "0.135.0" +cranelift-jit = "0.135.0" +cranelift-module = "0.135.0" +crc32fast = "1.3.2" +criterion = { package = "codspeed-criterion-compat", version = "5.0.1", features = ["html_reports"] } crossbeam-utils = "0.8.21" +csv-core = "0.1.11" +dns-lookup = "4.0" +exitcode = "1.1.2" flame = "0.2.2" -getrandom = { version = "0.3", features = ["std"] } +flamer = "0.5" +# Bump only when the openssl crate bumps it +foreign-types-shared = "0.1" +gethostname = "1.0.2" +getrandom = { version = "0.4", features = ["std"] } glob = "0.3" +half = "2" hex = "0.4.3" -indexmap = { version = "2.13.0", features = ["std"] } -insta = "1.46" -itertools = "0.14.0" +hexf-parse = "0.2.1" +indexmap = { version = "2.14.0", features = ["std"] } +insta = "1.47" +itertools = { version = "0.15.0", default-features = false, features = ["use_alloc"] } +itoa = "1" is-macro = "0.3.7" -junction = "1.4.1" -libc = "0.2.180" +jiff = "0.2" +js-sys = "0.3" +junction = "2.0.0" +lexical-parse-float = "1.0.6" +libc = "0.2.186" libffi = "5" -log = "0.4.29" -nix = { version = "0.30", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } -malachite-bigint = "0.9" -malachite-q = "0.9" -malachite-base = "0.9" -memchr = "2.7.4" +libloading = "0.9" +xz = "0.4.6-rc.0" +xz-sys = "0.4.6-rc.0" +libsqlite3-sys = "0.38" +libz-rs-sys = "0.6" +zlib-rs = { version = "=0.6.7", default-features = false, features = ["rust-allocator", "__internal-api"] } +lock_api = "0.4" +log = "0.4.30" +lz4_flex = "0.14" +nix = { version = "0.31", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } +mac_address = "1.1.3" +malachite-bigint = "0.11.0" +malachite-q = "0.11.0" +malachite-base = "0.11.0" +md-5 = "0.11" +memchr = { version = "2.8.1", default-features = false, features = ["alloc"] } +memmap2 = "0.9.10" +mt19937 = "3.3" num-complex = "0.4.6" num-integer = "0.1.46" num-traits = "0.2" +num_cpus = "1.17.0" num_enum = { version = "0.7", default-features = false } +openssl = "0.10.80" +openssl-sys = "0.9.110" optional = "0.5" -once_cell = "1.20.3" parking_lot = "0.12.3" paste = "1.0.15" +pem-rfc7468 = "1.0" +pkcs8 = "0.11" proc-macro2 = "1.0.105" -pymath = { version = "0.1.5", features = ["mul_add", "malachite-bigint", "complex"] } -quote = "1.0.44" +psm = "0.1" +pymath = { version = "0.2.0", features = ["mul_add", "malachite-bigint", "complex"] } +pyo3 = "0.29" +quote = "1.0.45" radium = "1.1.1" -rand = "0.9" -rand_core = { version = "0.9", features = ["os_rng"] } -rustix = { version = "1.1", features = ["event"] } -rustyline = "17.0.1" +rand = "0.10" +rapidhash = "4.4.1" +result-like = "0.5.0" +rustix = { version = "1.1", features = ["event", "fs", "param", "system"] } +rustls = { version = "0.23.39", default-features = false } +rustls-graviola = "0.4" +rustls-native-certs = "0.8" +rustls-pemfile = "2.2" +rustls-webpki = { version = "0.103", default-features = false, features = ["std"] } +rustyline = "18" serde = { package = "serde_core", version = "1.0.225", default-features = false, features = ["alloc"] } -schannel = "0.1.28" -scoped-tls = "1" +schannel = "0.1.29" scopeguard = "1" +scrypt = { version = "0.12", default-features = false } +serde-wasm-bindgen = "0.6.5" +sha1 = "0.11" +sha2 = "0.11" +sha3 = "0.12" +shake = "0.1" +siphasher = "1" +socket2 = "0.6.4" static_assertions = "1.1" -strum = "0.27" -strum_macros = "0.27" +strum = "0.28" +strum_macros = "0.28" syn = "2" +syn-ext = "0.5.0" +system-configuration = "0.8.0" +tcl-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0" } +textwrap = { version = "0.16.2", default-features = false } +termios = "0.3.3" thiserror = "2.0" -thread_local = "1.1.9" -unicode-casing = "0.1.1" -unic-char-property = "0.9.0" -unic-normal = "0.9.0" -unic-ucd-age = "0.9.0" -unic-ucd-bidi = "0.9.0" -unic-ucd-category = "0.9.0" -unic-ucd-ident = "0.9.0" -unicode_names2 = "2.0.0" -unicode-bidi-mirroring = "0.4" +thin-vec = "0.2.14" +tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0" } +icu_casemap = "2" +icu_locale = "2" +icu_properties = "2" +icu_normalizer = "2" +uuid = "1.23.2" +unicode_names2 = { version = "3", default-features = false, features = ["no_std"] } widestring = "1.2.0" windows-sys = "0.61.2" wasm-bindgen = "0.2.106" +wasm-bindgen-futures = "0.4" +web-sys = "0.3" +webpki-roots = "1.0" +which = "8" +x509-parser = "0.18" +xml = "1.3" +writeable = "0.6" # Lints @@ -222,13 +325,106 @@ wasm-bindgen = "0.2.106" unsafe_code = "allow" unsafe_op_in_unsafe_fn = "deny" elided_lifetimes_in_paths = "warn" +unreachable_pub = "warn" [workspace.lints.clippy] -# alloc_instead_of_core = "warn" -# std_instead_of_alloc = "warn" -# std_instead_of_core = "warn" -perf = "warn" -style = "warn" -complexity = "warn" -suspicious = "warn" -correctness = "warn" +correctness = { level = "warn", priority = -2 } +suspicious = { level = "warn", priority = -2 } +perf = { level = "warn", priority = -2 } +style = { level = "warn", priority = -2 } +complexity = { level = "warn", priority = -2 } +# pedantic = { level = "warn", priority = -2 } # TODO: Enable this + +missing_errors_doc = "allow" # Too many errors. No auto-fix available +missing_panics_doc = "allow" # Too many errors. No auto-fix available +match_same_arms = "allow" # Not always more readable +if_not_else = "allow" # Not always more readable +single_match_else = "allow" +similar_names = "allow" + +# restriction lints +alloc_instead_of_core = "warn" +cfg_not_test = "warn" +iter_over_hash_type = "warn" +redundant_test_prefix = "warn" +std_instead_of_alloc = "warn" +std_instead_of_core = "warn" +tests_outside_test_module = "warn" + +# nursery lints to enforce gradually +collection_is_never_read = "warn" +debug_assert_with_mut_call = "warn" +derive_partial_eq_without_eq = "warn" +imprecise_flops = "warn" +iter_on_empty_collections = "warn" +iter_on_single_items = "warn" +literal_string_with_formatting_args = "warn" +needless_collect = "warn" +or_fun_call = "warn" +redundant_clone = "warn" +search_is_some = "warn" +significant_drop_in_scrutinee = "warn" +single_option_map = "warn" +trait_duplication_in_bounds = "warn" +tuple_array_conversions = "warn" +type_repetition_in_bounds = "warn" +unnecessary_struct_initialization = "warn" +unused_peekable = "warn" +unused_rounding = "warn" +use_self = "warn" +useless_let_if_seq = "warn" +while_float = "warn" + +# pedantic lints to enforce gradually +assigning_clones = "warn" +bool_to_int_with_if = "warn" +checked_conversions = "warn" +cloned_instead_of_copied = "warn" +collapsible_else_if = "warn" +comparison_chain = "warn" +copy_iterator = "warn" +doc_link_with_quotes = "warn" +duration_suboptimal_units = "warn" +elidable_lifetime_names = "warn" +enum_glob_use = "warn" +explicit_deref_methods = "warn" +explicit_into_iter_loop = "warn" +explicit_iter_loop = "warn" +filter_map_next = "warn" +flat_map_option = "warn" +format_collect = "warn" +inconsistent_struct_constructor = "warn" +index_refutable_slice = "warn" +inefficient_to_string = "warn" +ip_constant = "warn" +iter_filter_is_ok = "warn" +iter_filter_is_some = "warn" +large_futures = "warn" +large_types_passed_by_value = "warn" +manual_assert = "warn" +manual_instant_elapsed = "warn" +manual_is_variant_and = "warn" +map_unwrap_or = "warn" +match_bool = "warn" +mismatching_type_param_order = "warn" +must_use_candidate = "warn" +mut_mut = "warn" +needless_bitwise_bool = "warn" +needless_for_each = "warn" +non_std_lazy_statics = "warn" +option_as_ref_cloned = "warn" +ptr_offset_by_literal = "warn" +range_minus_one = "warn" +range_plus_one = "warn" +redundant_else = "warn" +ref_option = "warn" +return_self_not_must_use = "warn" +same_functions_in_if_condition = "warn" +single_char_pattern = "warn" +trivially_copy_pass_by_ref = "warn" +unchecked_time_subtraction = "warn" +uninlined_format_args = "warn" +unnecessary_box_returns = "warn" +unnecessary_join = "warn" +unnecessary_wraps = "warn" +unnested_or_patterns = "warn" diff --git a/LICENSE b/LICENSE index e2aa2ed952e..78442e7c9b2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2025 RustPython Team +Copyright (c) 2026 RustPython Team Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Lib/_android_support.py b/Lib/_android_support.py index a439d03a144..320dab52acd 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -168,6 +168,13 @@ def write(self, prio, tag, message): # message. message = message.replace(b"\x00", b"\xc0\x80") + # On API level 30 and higher, Logcat will strip any number of leading + # newlines. This is visible in all `logcat` modes, even --binary. Work + # around this by adding a leading space, which shouldn't make any + # difference to the log's usability. + if message.startswith(b"\n"): + message = b" " + message + with self._lock: now = time() self._bucket_level += ( diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 241d40d5740..996b094377a 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -461,8 +461,8 @@ def __subclasshook__(cls, C): class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. - This sets ``__args__`` to a tuple containing the flattened ``argtypes`` - followed by ``resulttype``. + This sets ``__args__`` to a tuple containing the flattened + ``argtypes`` followed by ``resulttype``. Example: ``Callable[[int, str], float]`` sets ``__args__`` to ``(int, str, float)``. @@ -927,8 +927,9 @@ def __delitem__(self, key): __marker = object() def pop(self, key, default=__marker): - '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding + value. If key is not found, d is returned if given, otherwise + KeyError is raised. ''' try: value = self[key] @@ -962,9 +963,12 @@ def clear(self): def update(self, other=(), /, **kwds): ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. - If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k] - If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v - In either case, this is followed by: for k, v in F.items(): D[k] = v + If E present and has a .keys() method, does: + for k in E.keys(): D[k] = E[k] + If E present and lacks .keys() method, does: + for (k, v) in E: D[k] = v + In either case, this is followed by: + for k, v in F.items(): D[k] = v ''' if isinstance(other, Mapping): for key in other: @@ -1029,8 +1033,8 @@ def __reversed__(self): yield self[i] def index(self, value, start=0, stop=None): - '''S.index(value, [start, [stop]]) -> integer -- return first index of value. - Raises ValueError if the value is not present. + '''S.index(value, [start, [stop]]) -> integer -- return first index of + value. Raises ValueError if the value is not present. Supporting start and stop arguments is optional, but recommended. @@ -1138,15 +1142,16 @@ def reverse(self): self[i], self[n-i-1] = self[n-i-1], self[i] def extend(self, values): - 'S.extend(iterable) -- extend sequence by appending elements from the iterable' + """S.extend(iterable) -- extend sequence by appending elements from the + iterable""" if values is self: values = list(values) for v in values: self.append(v) def pop(self, index=-1): - '''S.pop([index]) -> item -- remove and return item at index (default last). - Raise IndexError if list is empty or index is out of range. + '''S.pop([index]) -> item -- remove and return item at index (default + last). Raise IndexError if list is empty or index is out of range. ''' v = self[index] del self[index] diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index 17b9010278f..60793c391ae 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -22,7 +22,6 @@ 'tkMessageBox': 'tkinter.messagebox', 'ScrolledText': 'tkinter.scrolledtext', 'Tkconstants': 'tkinter.constants', - 'Tix': 'tkinter.tix', 'ttk': 'tkinter.ttk', 'Tkinter': 'tkinter', 'markupbase': '_markupbase', @@ -257,3 +256,4 @@ for excname in PYTHON3_IMPORTERROR_EXCEPTIONS: REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'ImportError') +del excname diff --git a/Lib/_markupbase.py b/Lib/_markupbase.py index 3ad7e279960..614f0cd16dd 100644 --- a/Lib/_markupbase.py +++ b/Lib/_markupbase.py @@ -13,7 +13,7 @@ _markedsectionclose = re.compile(r']\s*]\s*>') # An analysis of the MS-Word extensions is available at -# http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf +# http://web.archive.org/web/20060321153828/http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf _msmarkedsectionclose = re.compile(r']\s*>') diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index bb55ee423cf..001a016621c 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -1,16 +1,223 @@ -# This file is generated by scripts/generate_opcode_metadata.py -# for RustPython bytecode format (CPython 3.13 compatible opcode numbers). +# This file is generated by tools/opcode_metadata/generate_py_opcode_metadata.py +# for RustPython bytecode format (CPython 3.14 compatible opcode numbers). # Do not edit! -_specializations = {} +_specializations = { + "RESUME": [ + "RESUME_CHECK", + ], + "LOAD_CONST": [ + "LOAD_CONST_MORTAL", + "LOAD_CONST_IMMORTAL", + ], + "TO_BOOL": [ + "TO_BOOL_ALWAYS_TRUE", + "TO_BOOL_BOOL", + "TO_BOOL_INT", + "TO_BOOL_LIST", + "TO_BOOL_NONE", + "TO_BOOL_STR", + ], + "BINARY_OP": [ + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_ADD_INT", + "BINARY_OP_SUBTRACT_INT", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_SUBTRACT_FLOAT", + "BINARY_OP_ADD_UNICODE", + "BINARY_OP_SUBSCR_LIST_INT", + "BINARY_OP_SUBSCR_LIST_SLICE", + "BINARY_OP_SUBSCR_TUPLE_INT", + "BINARY_OP_SUBSCR_STR_INT", + "BINARY_OP_SUBSCR_DICT", + "BINARY_OP_SUBSCR_GETITEM", + "BINARY_OP_EXTEND", + "BINARY_OP_INPLACE_ADD_UNICODE", + ], + "STORE_SUBSCR": [ + "STORE_SUBSCR_DICT", + "STORE_SUBSCR_LIST_INT", + ], + "SEND": [ + "SEND_GEN", + ], + "UNPACK_SEQUENCE": [ + "UNPACK_SEQUENCE_TWO_TUPLE", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_LIST", + ], + "STORE_ATTR": [ + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", + ], + "LOAD_GLOBAL": [ + "LOAD_GLOBAL_MODULE", + "LOAD_GLOBAL_BUILTIN", + ], + "LOAD_SUPER_ATTR": [ + "LOAD_SUPER_ATTR_ATTR", + "LOAD_SUPER_ATTR_METHOD", + ], + "LOAD_ATTR": [ + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_MODULE", + "LOAD_ATTR_WITH_HINT", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_CLASS", + "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK", + "LOAD_ATTR_PROPERTY", + "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + "LOAD_ATTR_METHOD_WITH_VALUES", + "LOAD_ATTR_METHOD_NO_DICT", + "LOAD_ATTR_METHOD_LAZY_DICT", + "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + ], + "COMPARE_OP": [ + "COMPARE_OP_FLOAT", + "COMPARE_OP_INT", + "COMPARE_OP_STR", + ], + "CONTAINS_OP": [ + "CONTAINS_OP_SET", + "CONTAINS_OP_DICT", + ], + "JUMP_BACKWARD": [ + "JUMP_BACKWARD_NO_JIT", + "JUMP_BACKWARD_JIT", + ], + "FOR_ITER": [ + "FOR_ITER_LIST", + "FOR_ITER_TUPLE", + "FOR_ITER_RANGE", + "FOR_ITER_GEN", + ], + "CALL": [ + "CALL_BOUND_METHOD_EXACT_ARGS", + "CALL_PY_EXACT_ARGS", + "CALL_TYPE_1", + "CALL_STR_1", + "CALL_TUPLE_1", + "CALL_BUILTIN_CLASS", + "CALL_BUILTIN_O", + "CALL_BUILTIN_FAST", + "CALL_BUILTIN_FAST_WITH_KEYWORDS", + "CALL_LEN", + "CALL_ISINSTANCE", + "CALL_LIST_APPEND", + "CALL_METHOD_DESCRIPTOR_O", + "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + "CALL_METHOD_DESCRIPTOR_NOARGS", + "CALL_METHOD_DESCRIPTOR_FAST", + "CALL_ALLOC_AND_ENTER_INIT", + "CALL_PY_GENERAL", + "CALL_BOUND_METHOD_GENERAL", + "CALL_NON_PY_GENERAL", + ], + "CALL_KW": [ + "CALL_KW_BOUND_METHOD", + "CALL_KW_PY", + "CALL_KW_NON_PY", + ], +} -_specialized_opmap = {} +_specialized_opmap = { + 'BINARY_OP_ADD_FLOAT': 129, + 'BINARY_OP_ADD_INT': 130, + 'BINARY_OP_ADD_UNICODE': 131, + 'BINARY_OP_EXTEND': 132, + 'BINARY_OP_INPLACE_ADD_UNICODE': 3, + 'BINARY_OP_MULTIPLY_FLOAT': 133, + 'BINARY_OP_MULTIPLY_INT': 134, + 'BINARY_OP_SUBSCR_DICT': 135, + 'BINARY_OP_SUBSCR_GETITEM': 136, + 'BINARY_OP_SUBSCR_LIST_INT': 137, + 'BINARY_OP_SUBSCR_LIST_SLICE': 138, + 'BINARY_OP_SUBSCR_STR_INT': 139, + 'BINARY_OP_SUBSCR_TUPLE_INT': 140, + 'BINARY_OP_SUBTRACT_FLOAT': 141, + 'BINARY_OP_SUBTRACT_INT': 142, + 'CALL_ALLOC_AND_ENTER_INIT': 143, + 'CALL_BOUND_METHOD_EXACT_ARGS': 144, + 'CALL_BOUND_METHOD_GENERAL': 145, + 'CALL_BUILTIN_CLASS': 146, + 'CALL_BUILTIN_FAST': 147, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 148, + 'CALL_BUILTIN_O': 149, + 'CALL_ISINSTANCE': 150, + 'CALL_KW_BOUND_METHOD': 151, + 'CALL_KW_NON_PY': 152, + 'CALL_KW_PY': 153, + 'CALL_LEN': 154, + 'CALL_LIST_APPEND': 155, + 'CALL_METHOD_DESCRIPTOR_FAST': 156, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 157, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 158, + 'CALL_METHOD_DESCRIPTOR_O': 159, + 'CALL_NON_PY_GENERAL': 160, + 'CALL_PY_EXACT_ARGS': 161, + 'CALL_PY_GENERAL': 162, + 'CALL_STR_1': 163, + 'CALL_TUPLE_1': 164, + 'CALL_TYPE_1': 165, + 'COMPARE_OP_FLOAT': 166, + 'COMPARE_OP_INT': 167, + 'COMPARE_OP_STR': 168, + 'CONTAINS_OP_DICT': 169, + 'CONTAINS_OP_SET': 170, + 'FOR_ITER_GEN': 171, + 'FOR_ITER_LIST': 172, + 'FOR_ITER_RANGE': 173, + 'FOR_ITER_TUPLE': 174, + 'JUMP_BACKWARD_JIT': 175, + 'JUMP_BACKWARD_NO_JIT': 176, + 'LOAD_ATTR_CLASS': 177, + 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 178, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 179, + 'LOAD_ATTR_INSTANCE_VALUE': 180, + 'LOAD_ATTR_METHOD_LAZY_DICT': 181, + 'LOAD_ATTR_METHOD_NO_DICT': 182, + 'LOAD_ATTR_METHOD_WITH_VALUES': 183, + 'LOAD_ATTR_MODULE': 184, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 185, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 186, + 'LOAD_ATTR_PROPERTY': 187, + 'LOAD_ATTR_SLOT': 188, + 'LOAD_ATTR_WITH_HINT': 189, + 'LOAD_CONST_IMMORTAL': 190, + 'LOAD_CONST_MORTAL': 191, + 'LOAD_GLOBAL_BUILTIN': 192, + 'LOAD_GLOBAL_MODULE': 193, + 'LOAD_SUPER_ATTR_ATTR': 194, + 'LOAD_SUPER_ATTR_METHOD': 195, + 'RESUME_CHECK': 196, + 'SEND_GEN': 197, + 'STORE_ATTR_INSTANCE_VALUE': 198, + 'STORE_ATTR_SLOT': 199, + 'STORE_ATTR_WITH_HINT': 200, + 'STORE_SUBSCR_DICT': 201, + 'STORE_SUBSCR_LIST_INT': 202, + 'TO_BOOL_ALWAYS_TRUE': 203, + 'TO_BOOL_BOOL': 204, + 'TO_BOOL_INT': 205, + 'TO_BOOL_LIST': 206, + 'TO_BOOL_NONE': 207, + 'TO_BOOL_STR': 208, + 'UNPACK_SEQUENCE_LIST': 209, + 'UNPACK_SEQUENCE_TUPLE': 210, + 'UNPACK_SEQUENCE_TWO_TUPLE': 211, +} opmap = { 'CACHE': 0, + 'RESERVED': 17, + 'RESUME': 128, + 'INSTRUMENTED_LINE': 254, + 'ENTER_EXECUTOR': 255, 'BINARY_SLICE': 1, 'BUILD_TEMPLATE': 2, - 'BINARY_OP_INPLACE_ADD_UNICODE': 3, 'CALL_FUNCTION_EX': 4, 'CHECK_EG_MATCH': 5, 'CHECK_EXC_MATCH': 6, @@ -24,7 +231,6 @@ 'GET_AITER': 14, 'GET_ANEXT': 15, 'GET_ITER': 16, - 'RESERVED': 17, 'GET_LEN': 18, 'GET_YIELD_FROM_ITER': 19, 'INTERPRETER_EXIT': 20, @@ -128,90 +334,6 @@ 'UNPACK_EX': 118, 'UNPACK_SEQUENCE': 119, 'YIELD_VALUE': 120, - 'RESUME': 128, - 'BINARY_OP_ADD_FLOAT': 129, - 'BINARY_OP_ADD_INT': 130, - 'BINARY_OP_ADD_UNICODE': 131, - 'BINARY_OP_EXTEND': 132, - 'BINARY_OP_MULTIPLY_FLOAT': 133, - 'BINARY_OP_MULTIPLY_INT': 134, - 'BINARY_OP_SUBSCR_DICT': 135, - 'BINARY_OP_SUBSCR_GETITEM': 136, - 'BINARY_OP_SUBSCR_LIST_INT': 137, - 'BINARY_OP_SUBSCR_LIST_SLICE': 138, - 'BINARY_OP_SUBSCR_STR_INT': 139, - 'BINARY_OP_SUBSCR_TUPLE_INT': 140, - 'BINARY_OP_SUBTRACT_FLOAT': 141, - 'BINARY_OP_SUBTRACT_INT': 142, - 'CALL_ALLOC_AND_ENTER_INIT': 143, - 'CALL_BOUND_METHOD_EXACT_ARGS': 144, - 'CALL_BOUND_METHOD_GENERAL': 145, - 'CALL_BUILTIN_CLASS': 146, - 'CALL_BUILTIN_FAST': 147, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 148, - 'CALL_BUILTIN_O': 149, - 'CALL_ISINSTANCE': 150, - 'CALL_KW_BOUND_METHOD': 151, - 'CALL_KW_NON_PY': 152, - 'CALL_KW_PY': 153, - 'CALL_LEN': 154, - 'CALL_LIST_APPEND': 155, - 'CALL_METHOD_DESCRIPTOR_FAST': 156, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 157, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 158, - 'CALL_METHOD_DESCRIPTOR_O': 159, - 'CALL_NON_PY_GENERAL': 160, - 'CALL_PY_EXACT_ARGS': 161, - 'CALL_PY_GENERAL': 162, - 'CALL_STR_1': 163, - 'CALL_TUPLE_1': 164, - 'CALL_TYPE_1': 165, - 'COMPARE_OP_FLOAT': 166, - 'COMPARE_OP_INT': 167, - 'COMPARE_OP_STR': 168, - 'CONTAINS_OP_DICT': 169, - 'CONTAINS_OP_SET': 170, - 'FOR_ITER_GEN': 171, - 'FOR_ITER_LIST': 172, - 'FOR_ITER_RANGE': 173, - 'FOR_ITER_TUPLE': 174, - 'JUMP_BACKWARD_JIT': 175, - 'JUMP_BACKWARD_NO_JIT': 176, - 'LOAD_ATTR_CLASS': 177, - 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 178, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 179, - 'LOAD_ATTR_INSTANCE_VALUE': 180, - 'LOAD_ATTR_METHOD_LAZY_DICT': 181, - 'LOAD_ATTR_METHOD_NO_DICT': 182, - 'LOAD_ATTR_METHOD_WITH_VALUES': 183, - 'LOAD_ATTR_MODULE': 184, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 185, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 186, - 'LOAD_ATTR_PROPERTY': 187, - 'LOAD_ATTR_SLOT': 188, - 'LOAD_ATTR_WITH_HINT': 189, - 'LOAD_CONST_IMMORTAL': 190, - 'LOAD_CONST_MORTAL': 191, - 'LOAD_GLOBAL_BUILTIN': 192, - 'LOAD_GLOBAL_MODULE': 193, - 'LOAD_SUPER_ATTR_ATTR': 194, - 'LOAD_SUPER_ATTR_METHOD': 195, - 'RESUME_CHECK': 196, - 'SEND_GEN': 197, - 'STORE_ATTR_INSTANCE_VALUE': 198, - 'STORE_ATTR_SLOT': 199, - 'STORE_ATTR_WITH_HINT': 200, - 'STORE_SUBSCR_DICT': 201, - 'STORE_SUBSCR_LIST_INT': 202, - 'TO_BOOL_ALWAYS_TRUE': 203, - 'TO_BOOL_BOOL': 204, - 'TO_BOOL_INT': 205, - 'TO_BOOL_LIST': 206, - 'TO_BOOL_NONE': 207, - 'TO_BOOL_STR': 208, - 'UNPACK_SEQUENCE_LIST': 209, - 'UNPACK_SEQUENCE_TUPLE': 210, - 'UNPACK_SEQUENCE_TWO_TUPLE': 211, 'INSTRUMENTED_END_FOR': 234, 'INSTRUMENTED_POP_ITER': 235, 'INSTRUMENTED_END_SEND': 236, @@ -232,8 +354,6 @@ 'INSTRUMENTED_CALL_KW': 251, 'INSTRUMENTED_CALL_FUNCTION_EX': 252, 'INSTRUMENTED_JUMP_BACKWARD': 253, - 'INSTRUMENTED_LINE': 254, - 'ENTER_EXECUTOR': 255, 'ANNOTATIONS_PLACEHOLDER': 256, 'JUMP': 257, 'JUMP_IF_FALSE': 258, @@ -247,6 +367,5 @@ 'STORE_FAST_MAYBE_NULL': 266, } -# CPython 3.13 compatible: opcodes < 44 have no argument -HAVE_ARGUMENT = 44 -MIN_INSTRUMENTED_OPCODE = 236 +HAVE_ARGUMENT = 43 +MIN_INSTRUMENTED_OPCODE = 234 diff --git a/Lib/_pycodecs.py b/Lib/_pycodecs.py index d0efa9ad6bb..02886579111 100644 --- a/Lib/_pycodecs.py +++ b/Lib/_pycodecs.py @@ -22,10 +22,10 @@ The builtin Unicode codecs use the following interface: - _encode(Unicode_object[,errors='strict']) -> + _encode(Unicode_object[,errors='strict']) -> (string object, bytes consumed) - _decode(char_buffer_obj[,errors='strict']) -> + _decode(char_buffer_obj[,errors='strict']) -> (Unicode object, bytes consumed) _encode() interfaces also accept non-Unicode object as @@ -44,47 +44,76 @@ From PyPy v1.0.0 """ -#from unicodecodec import * - -__all__ = ['register', 'lookup', 'lookup_error', 'register_error', 'encode', 'decode', - 'latin_1_encode', 'mbcs_decode', 'readbuffer_encode', 'escape_encode', - 'utf_8_decode', 'raw_unicode_escape_decode', 'utf_7_decode', - 'unicode_escape_encode', 'latin_1_decode', 'utf_16_decode', - 'unicode_escape_decode', 'ascii_decode', 'charmap_encode', 'charmap_build', - 'unicode_internal_encode', 'unicode_internal_decode', 'utf_16_ex_decode', - 'escape_decode', 'charmap_decode', 'utf_7_encode', 'mbcs_encode', - 'ascii_encode', 'utf_16_encode', 'raw_unicode_escape_encode', 'utf_8_encode', - 'utf_16_le_encode', 'utf_16_be_encode', 'utf_16_le_decode', 'utf_16_be_decode',] +# from unicodecodec import * + +__all__ = [ + "register", + "lookup", + "lookup_error", + "register_error", + "encode", + "decode", + "latin_1_encode", + "mbcs_decode", + "readbuffer_encode", + "escape_encode", + "utf_8_decode", + "raw_unicode_escape_decode", + "utf_7_decode", + "unicode_escape_encode", + "latin_1_decode", + "utf_16_decode", + "unicode_escape_decode", + "ascii_decode", + "charmap_encode", + "charmap_build", + "unicode_internal_encode", + "unicode_internal_decode", + "utf_16_ex_decode", + "escape_decode", + "charmap_decode", + "utf_7_encode", + "mbcs_encode", + "ascii_encode", + "utf_16_encode", + "raw_unicode_escape_encode", + "utf_8_encode", + "utf_16_le_encode", + "utf_16_be_encode", + "utf_16_le_decode", + "utf_16_be_decode", + "utf_32_ex_decode", +] import sys import warnings from _codecs import * -def latin_1_encode( obj, errors='strict'): - """None - """ +def latin_1_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeLatin1(obj, len(obj), errors) res = bytes(res) return res, len(obj) + + # XXX MBCS codec might involve ctypes ? def mbcs_decode(): - """None - """ + """None""" pass -def readbuffer_encode( obj, errors='strict'): - """None - """ + +def readbuffer_encode(obj, errors="strict"): + """None""" if isinstance(obj, str): res = obj.encode() else: res = bytes(obj) return res, len(obj) -def escape_encode( obj, errors='strict'): - """None - """ + +def escape_encode(obj, errors="strict"): + """None""" if not isinstance(obj, bytes): raise TypeError("must be bytes") s = repr(obj).encode() @@ -93,85 +122,88 @@ def escape_encode( obj, errors='strict'): v = v.replace(b"'", b"\\'").replace(b'\\"', b'"') return v, len(obj) -def raw_unicode_escape_decode( data, errors='strict', final=False): - """None - """ - res = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors, final) - res = ''.join(res) - return res, len(data) -def utf_7_decode( data, errors='strict'): - """None - """ - res = PyUnicode_DecodeUTF7(data, len(data), errors) - res = ''.join(res) - return res, len(data) +def raw_unicode_escape_decode(data, errors="strict", final=True): + """None""" + res, consumed = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors, final) + res = "".join(res) + return res, consumed -def unicode_escape_encode( obj, errors='strict'): - """None - """ + +def utf_7_decode(data, errors="strict", final=False): + """None""" + res, consumed = PyUnicode_DecodeUTF7(data, len(data), errors, final) + res = "".join(res) + return res, consumed + + +def unicode_escape_encode(obj, errors="strict"): + """None""" res = unicodeescape_string(obj, len(obj), 0) - res = b''.join(res) + res = b"".join(res) return res, len(obj) -def latin_1_decode( data, errors='strict'): - """None - """ + +def latin_1_decode(data, errors="strict"): + """None""" res = PyUnicode_DecodeLatin1(data, len(data), errors) - res = ''.join(res) + res = "".join(res) return res, len(data) -def utf_16_decode( data, errors='strict', final=False): - """None - """ + +def utf_16_decode(data, errors="strict", final=False): + """None""" consumed = len(data) if final: consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'native', final) - res = ''.join(res) + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, "native", final + ) + res = "".join(res) return res, consumed -def unicode_escape_decode( data, errors='strict', final=False): - """None - """ - res = PyUnicode_DecodeUnicodeEscape(data, len(data), errors, final) - res = ''.join(res) - return res, len(data) +def unicode_escape_decode(data, errors="strict", final=True): + """None""" + res, consumed = PyUnicode_DecodeUnicodeEscape(data, len(data), errors, final) + res = "".join(res) + return res, consumed -def ascii_decode( data, errors='strict'): - """None - """ + +def ascii_decode(data, errors="strict"): + """None""" res = PyUnicode_DecodeASCII(data, len(data), errors) - res = ''.join(res) + res = "".join(res) return res, len(data) -def charmap_encode(obj, errors='strict', mapping='latin-1'): - """None - """ + +def charmap_encode(obj, errors="strict", mapping="latin-1"): + """None""" res = PyUnicode_EncodeCharmap(obj, len(obj), mapping, errors) res = bytes(res) return res, len(obj) + def charmap_build(s): return {ord(c): i for i, c in enumerate(s)} + if sys.maxunicode == 65535: unicode_bytes = 2 else: unicode_bytes = 4 -def unicode_internal_encode( obj, errors='strict'): - """None - """ + +def unicode_internal_encode(obj, errors="strict"): + """None""" if type(obj) == str: p = bytearray() t = [ord(x) for x in obj] for i in t: b = bytearray() for j in range(unicode_bytes): - b.append(i%256) + b.append(i % 256) i >>= 8 if sys.byteorder == "big": b.reverse() @@ -179,12 +211,12 @@ def unicode_internal_encode( obj, errors='strict'): res = bytes(p) return res, len(res) else: - res = "You can do better than this" # XXX make this right + res = "You can do better than this" # XXX make this right return res, len(res) -def unicode_internal_decode( unistr, errors='strict'): - """None - """ + +def unicode_internal_decode(unistr, errors="strict"): + """None""" if type(unistr) == str: return unistr, len(unistr) else: @@ -198,165 +230,437 @@ def unicode_internal_decode( unistr, errors='strict'): start = 0 stop = unicode_bytes step = 1 - while i < len(unistr)-unicode_bytes+1: + while i < len(unistr) - unicode_bytes + 1: t = 0 h = 0 for j in range(start, stop, step): - t += ord(unistr[i+j])<<(h*8) + t += ord(unistr[i + j]) << (h * 8) h += 1 i += unicode_bytes p += chr(t) - res = ''.join(p) + res = "".join(p) return res, len(res) -def utf_16_ex_decode( data, errors='strict', byteorder=0, final=0): - """None - """ + +def utf_16_ex_decode(data, errors="strict", byteorder=0, final=0): + """None""" if byteorder == 0: - bm = 'native' + bm = "native" elif byteorder == -1: - bm = 'little' + bm = "little" else: - bm = 'big' + bm = "big" consumed = len(data) if final: consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, bm, final) - res = ''.join(res) + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, bm, final + ) + res = "".join(res) return res, consumed, byteorder -# XXX needs error messages when the input is invalid -def escape_decode(data, errors='strict'): - """None - """ + +def utf_32_ex_decode(data, errors="strict", byteorder=0, final=0): + """None""" + if byteorder == 0: + if len(data) < 4: + if final and len(data): + if sys.byteorder == "little": + bm = "little" + else: + bm = "big" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, bm, final + ) + return "".join(res), consumed, 0 + return "", 0, 0 + if data[0:4] == b"\xff\xfe\x00\x00": + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "little", final + ) + return "".join(res), consumed + 4, -1 + if data[0:4] == b"\x00\x00\xfe\xff": + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "big", final + ) + return "".join(res), consumed + 4, 1 + if sys.byteorder == "little": + bm = "little" + else: + bm = "big" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, bm, final + ) + return "".join(res), consumed, 0 + + if byteorder == -1: + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "little", final + ) + return "".join(res), consumed, -1 + + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "big", final + ) + return "".join(res), consumed, 1 + + +def _is_hex_digit(b): + return ( + 0x30 <= b <= 0x39 # 0-9 + or 0x41 <= b <= 0x46 # A-F + or 0x61 <= b <= 0x66 + ) # a-f + + +def escape_decode(data, errors="strict"): + if isinstance(data, str): + data = data.encode("latin-1") l = len(data) i = 0 res = bytearray() while i < l: - - if data[i] == '\\': + if data[i] == 0x5C: # '\\' i += 1 if i >= l: raise ValueError("Trailing \\ in string") - else: - if data[i] == '\\': - res += b'\\' - elif data[i] == 'n': - res += b'\n' - elif data[i] == 't': - res += b'\t' - elif data[i] == 'r': - res += b'\r' - elif data[i] == 'b': - res += b'\b' - elif data[i] == '\'': - res += b'\'' - elif data[i] == '\"': - res += b'\"' - elif data[i] == 'f': - res += b'\f' - elif data[i] == 'a': - res += b'\a' - elif data[i] == 'v': - res += b'\v' - elif '0' <= data[i] <= '9': - # emulate a strange wrap-around behavior of CPython: - # \400 is the same as \000 because 0400 == 256 - octal = data[i:i+3] - res.append(int(octal, 8) & 0xFF) - i += 2 - elif data[i] == 'x': - hexa = data[i+1:i+3] - res.append(int(hexa, 16)) + ch = data[i] + if ch == 0x5C: + res.append(0x5C) # \\ + elif ch == 0x27: + res.append(0x27) # \' + elif ch == 0x22: + res.append(0x22) # \" + elif ch == 0x61: + res.append(0x07) # \a + elif ch == 0x62: + res.append(0x08) # \b + elif ch == 0x66: + res.append(0x0C) # \f + elif ch == 0x6E: + res.append(0x0A) # \n + elif ch == 0x72: + res.append(0x0D) # \r + elif ch == 0x74: + res.append(0x09) # \t + elif ch == 0x76: + res.append(0x0B) # \v + elif ch == 0x0A: + pass # \ continuation + elif 0x30 <= ch <= 0x37: # \0-\7 octal + val = ch - 0x30 + if i + 1 < l and 0x30 <= data[i + 1] <= 0x37: + i += 1 + val = (val << 3) | (data[i] - 0x30) + if i + 1 < l and 0x30 <= data[i + 1] <= 0x37: + i += 1 + val = (val << 3) | (data[i] - 0x30) + res.append(val & 0xFF) + elif ch == 0x78: # \x hex + hex_count = 0 + for j in range(1, 3): + if i + j < l and _is_hex_digit(data[i + j]): + hex_count += 1 + else: + break + if hex_count < 2: + if errors == "strict": + raise ValueError("invalid \\x escape at position %d" % (i - 1)) + elif errors == "replace": + res.append(0x3F) # '?' + i += hex_count + else: + res.append(int(bytes(data[i + 1 : i + 3]), 16)) i += 2 + else: + import warnings + + warnings.warn( + '"\\%c" is an invalid escape sequence' % ch + if 0x20 <= ch < 0x7F + else '"\\x%02x" is an invalid escape sequence' % ch, + DeprecationWarning, + stacklevel=2, + ) + res.append(0x5C) + res.append(ch) else: res.append(data[i]) i += 1 - res = bytes(res) - return res, len(res) + return bytes(res), l -def charmap_decode( data, errors='strict', mapping=None): - """None - """ + +def charmap_decode(data, errors="strict", mapping=None): + """None""" res = PyUnicode_DecodeCharmap(data, len(data), mapping, errors) - res = ''.join(res) + res = "".join(res) return res, len(data) -def utf_7_encode( obj, errors='strict'): - """None - """ +def utf_7_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeUTF7(obj, len(obj), 0, 0, errors) - res = b''.join(res) + res = b"".join(res) return res, len(obj) -def mbcs_encode( obj, errors='strict'): - """None - """ + +def mbcs_encode(obj, errors="strict"): + """None""" pass + + ## return (PyUnicode_EncodeMBCS( -## (obj), +## (obj), ## len(obj), ## errors), ## len(obj)) - -def ascii_encode( obj, errors='strict'): - """None - """ + +def ascii_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeASCII(obj, len(obj), errors) res = bytes(res) return res, len(obj) -def utf_16_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'native') + +def utf_16_encode(obj, errors="strict"): + """None""" + res = PyUnicode_EncodeUTF16(obj, len(obj), errors, "native") res = bytes(res) return res, len(obj) -def raw_unicode_escape_encode( obj, errors='strict'): - """None - """ + +def raw_unicode_escape_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeRawUnicodeEscape(obj, len(obj)) res = bytes(res) return res, len(obj) -def utf_16_le_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'little') + +def utf_16_le_encode(obj, errors="strict"): + """None""" + res = PyUnicode_EncodeUTF16(obj, len(obj), errors, "little") res = bytes(res) return res, len(obj) -def utf_16_be_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'big') + +def utf_16_be_encode(obj, errors="strict"): + """None""" + res = PyUnicode_EncodeUTF16(obj, len(obj), errors, "big") res = bytes(res) return res, len(obj) -def utf_16_le_decode( data, errors='strict', byteorder=0, final = 0): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'little', final) - res = ''.join(res) + +def utf_16_le_decode(data, errors="strict", final=0): + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, "little", final + ) + res = "".join(res) + return res, consumed + + +def utf_16_be_decode(data, errors="strict", final=0): + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, "big", final + ) + res = "".join(res) return res, consumed -def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): - """None + +def STORECHAR32(ch, byteorder): + """Store a 32-bit character as 4 bytes in the specified byte order.""" + b0 = ch & 0xFF + b1 = (ch >> 8) & 0xFF + b2 = (ch >> 16) & 0xFF + b3 = (ch >> 24) & 0xFF + if byteorder == "little": + return [b0, b1, b2, b3] + else: # big-endian + return [b3, b2, b1, b0] + + +def _encoded_replacement(replacement, store, unit_bytes, encoding, s, pos): + """The bytes an error handler's replacement adds to the output. + + A str is encoded the way the rest of the input is. A bytes object is taken + as it stands, and only when it fills whole code units. """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'big', final) - res = ''.join(res) + if isinstance(replacement, bytes): + if len(replacement) % unit_bytes: + raise UnicodeEncodeError( + encoding, s, pos, pos + 1, "surrogates not allowed" + ) + return list(replacement) + encoded = [] + for c in replacement: + encoded += store(ord(c)) + return encoded + + +def PyUnicode_EncodeUTF32(s, size, errors, byteorder="little"): + """Encode a Unicode string to UTF-32.""" + p = [] + bom = sys.byteorder + + if byteorder == "native": + bom = sys.byteorder + # Add BOM for native encoding + p += STORECHAR32(0xFEFF, bom) + + if byteorder == "little": + bom = "little" + elif byteorder == "big": + bom = "big" + + # Only the mode that writes a BOM leaves the order out of its name. + encoding = "utf-32" if byteorder == "native" else "utf-32-" + bom[0] + "e" + + def store(cp): + return STORECHAR32(cp, bom) + + pos = 0 + while pos < len(s): + ch = ord(s[pos]) + if 0xD800 <= ch <= 0xDFFF and errors != "surrogatepass": + start = pos + res, pos = unicode_call_errorhandler( + errors, encoding, "surrogates not allowed", s, pos, pos + 1, False + ) + p += _encoded_replacement(res, store, 4, encoding, s, start) + else: + p += store(ch) + pos += 1 + + return p + + +def utf_32_encode(obj, errors="strict"): + """UTF-32 encoding with BOM.""" + encoded = PyUnicode_EncodeUTF32(obj, len(obj), errors, "native") + return bytes(encoded), len(obj) + + +def utf_32_le_encode(obj, errors="strict"): + """UTF-32 little-endian encoding without BOM.""" + encoded = PyUnicode_EncodeUTF32(obj, len(obj), errors, "little") + return bytes(encoded), len(obj) + + +def utf_32_be_encode(obj, errors="strict"): + """UTF-32 big-endian encoding without BOM.""" + res = PyUnicode_EncodeUTF32(obj, len(obj), errors, "big") + res = bytes(res) + return res, len(obj) + + +def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final=0): + """Decode UTF-32 encoded bytes to Unicode string.""" + if size == 0: + return [], 0, 0 + + result = [] + pos = 0 + aligned_size = (size // 4) * 4 + encoding = "utf-32-le" if byteorder == "little" else "utf-32-be" + + while pos + 3 < aligned_size: + if byteorder == "little": + ch = ( + data[pos] + | (data[pos + 1] << 8) + | (data[pos + 2] << 16) + | (data[pos + 3] << 24) + ) + else: # big-endian + ch = ( + (data[pos] << 24) + | (data[pos + 1] << 16) + | (data[pos + 2] << 8) + | data[pos + 3] + ) + + # Validate code point + if ch > 0x10FFFF: + res, pos = unicode_call_errorhandler( + errors, + encoding, + "code point not in range(0x110000)", + data, + pos, + pos + 4, + True, + ) + result.append(res) + elif 0xD800 <= ch <= 0xDFFF: + if errors == "surrogatepass": + result.append(chr(ch)) + pos += 4 + else: + msg = "code point in surrogate code point range(0xd800, 0xe000)" + res, pos = unicode_call_errorhandler( + errors, encoding, msg, data, pos, pos + 4, True + ) + result.append(res) + else: + result.append(chr(ch)) + pos += 4 + + # Handle trailing incomplete bytes + if pos < size: + if final: + res, pos = unicode_call_errorhandler( + errors, encoding, "truncated data", data, pos, size, True + ) + if res: + result.append(res) + + return result, pos, 0 + + +def utf_32_decode(data, errors="strict", final=0): + """UTF-32 decoding with BOM detection.""" + if len(data) >= 4: + # Check for BOM + if data[0:4] == b"\xff\xfe\x00\x00": + # UTF-32 LE BOM + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "little", final + ) + res = "".join(res) + return res, consumed + 4 + elif data[0:4] == b"\x00\x00\xfe\xff": + # UTF-32 BE BOM + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "big", final + ) + res = "".join(res) + return res, consumed + 4 + + # Default to little-endian if no BOM + byteorder = "little" if sys.byteorder == "little" else "big" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, byteorder, final + ) + res = "".join(res) return res, consumed +def utf_32_le_decode(data, errors="strict", final=0): + """UTF-32 little-endian decoding without BOM.""" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "little", final + ) + res = "".join(res) + return res, consumed + + +def utf_32_be_decode(data, errors="strict", final=0): + """UTF-32 big-endian decoding without BOM.""" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "big", final + ) + res = "".join(res) + return res, consumed # ---------------------------------------------------------------------- @@ -364,9 +668,9 @@ def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): ##import sys ##""" Python implementation of CPythons builtin unicode codecs. ## -## Generally the functions in this module take a list of characters an returns +## Generally the functions in this module take a list of characters an returns ## a list of characters. -## +## ## For use in the PyPy project""" @@ -376,283 +680,482 @@ def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): ## 1 - special ## 2 - whitespace (optional) ## 3 - RFC2152 Set O (optional) - + utf7_special = [ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, - 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, - 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 3, + 3, + 3, + 3, + 3, + 3, + 0, + 0, + 0, + 3, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 3, + 3, + 3, + 0, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 1, + 3, + 3, + 3, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 3, + 3, + 1, + 1, ] -unicode_latin1 = [None]*256 +unicode_latin1 = [None] * 256 def SPECIAL(c, encodeO, encodeWS): c = ord(c) - return (c>127 or utf7_special[c] == 1) or \ - (encodeWS and (utf7_special[(c)] == 2)) or \ - (encodeO and (utf7_special[(c)] == 3)) + return ( + (c > 127 or utf7_special[c] == 1) + or (encodeWS and (utf7_special[(c)] == 2)) + or (encodeO and (utf7_special[(c)] == 3)) + ) + + def B64(n): - return bytes([b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f]]) + return bytes( + [ + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[ + (n) & 0x3F + ] + ] + ) + + def B64CHAR(c): - return (c.isalnum() or (c) == b'+' or (c) == b'/') + return c.isalnum() or (c) == b"+" or (c) == b"/" + + def UB64(c): - if (c) == b'+' : - return 62 - elif (c) == b'/': - return 63 - elif (c) >= b'a': - return ord(c) - 71 - elif (c) >= b'A': - return ord(c) - 65 - else: + if (c) == b"+": + return 62 + elif (c) == b"/": + return 63 + elif (c) >= b"a": + return ord(c) - 71 + elif (c) >= b"A": + return ord(c) - 65 + else: return ord(c) + 4 -def ENCODE( ch, bits) : + +def ENCODE(ch, bits): out = [] - while (bits >= 6): - out += B64(ch >> (bits-6)) - bits -= 6 + while bits >= 6: + out += B64(ch >> (bits - 6)) + bits -= 6 return out, bits -def PyUnicode_DecodeUTF7(s, size, errors): - starts = s - errmsg = "" - inShift = 0 - bitsleft = 0 - charsleft = 0 - surrogate = 0 - p = [] - errorHandler = None - exc = None +def _IS_BASE64(ch): + return ( + (ord("A") <= ch <= ord("Z")) + or (ord("a") <= ch <= ord("z")) + or (ord("0") <= ch <= ord("9")) + or ch == ord("+") + or ch == ord("/") + ) + + +def _FROM_BASE64(ch): + if ch == ord("+"): + return 62 + if ch == ord("/"): + return 63 + if ch >= ord("a"): + return ch - 71 + if ch >= ord("A"): + return ch - 65 + if ch >= ord("0"): + return ch - ord("0") + 52 + return -1 + + +def _DECODE_DIRECT(ch): + return ch <= 127 and ch != ord("+") + - if (size == 0): - return '' +def PyUnicode_DecodeUTF7(s, size, errors, final=False): + if size == 0: + return [], 0 + + p = [] + inShift = False + base64bits = 0 + base64buffer = 0 + surrogate = 0 + startinpos = 0 + shiftOutStart = 0 i = 0 + while i < size: - - ch = bytes([s[i]]) - if (inShift): - if ((ch == b'-') or not B64CHAR(ch)): - inShift = 0 + ch = s[i] + if inShift: + if _IS_BASE64(ch): + base64buffer = (base64buffer << 6) | _FROM_BASE64(ch) + base64bits += 6 i += 1 - - while (bitsleft >= 16): - outCh = ((charsleft) >> (bitsleft-16)) & 0xffff - bitsleft -= 16 - - if (surrogate): - ## We have already generated an error for the high surrogate - ## so let's not bother seeing if the low surrogate is correct or not - surrogate = 0 - elif (0xDC00 <= (outCh) and (outCh) <= 0xDFFF): - ## This is a surrogate pair. Unfortunately we can't represent - ## it in a 16-bit character - surrogate = 1 - msg = "code pairs are not supported" - out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i) - p.append(out) - bitsleft = 0 - break + if base64bits >= 16: + outCh = (base64buffer >> (base64bits - 16)) & 0xFFFF + base64bits -= 16 + base64buffer &= (1 << base64bits) - 1 + if surrogate: + if 0xDC00 <= outCh <= 0xDFFF: + ch2 = ( + 0x10000 + + ((surrogate - 0xD800) << 10) + + (outCh - 0xDC00) + ) + p.append(chr(ch2)) + surrogate = 0 + continue + else: + p.append(chr(surrogate)) + surrogate = 0 + if 0xD800 <= outCh <= 0xDBFF: + surrogate = outCh else: - p.append(chr(outCh )) - #p += out - if (bitsleft >= 6): -## /* The shift sequence has a partial character in it. If -## bitsleft < 6 then we could just classify it as padding -## but that is not the case here */ - msg = "partial character in shift sequence" - out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i) - -## /* According to RFC2152 the remaining bits should be zero. We -## choose to signal an error/insert a replacement character -## here so indicate the potential of a misencoded character. */ - -## /* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */ -## if (bitsleft and (charsleft << (sizeof(charsleft) * 8 - bitsleft))): -## raise UnicodeDecodeError, "non-zero padding bits in shift sequence" - if (ch == b'-') : - if ((i < size) and (s[i] == '-')) : - p += '-' - inShift = 1 - - elif SPECIAL(ch, 0, 0) : - raise UnicodeDecodeError("unexpected special character") - - else: - p.append(chr(ord(ch))) + p.append(chr(outCh)) else: - charsleft = (charsleft << 6) | UB64(ch) - bitsleft += 6 - i += 1 -## /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate); - elif ( ch == b'+' ): + inShift = False + if base64bits > 0: + if base64bits >= 6: + i += 1 + errmsg = "partial character in shift sequence" + out, i = unicode_call_errorhandler( + errors, "utf7", errmsg, s, startinpos, i + ) + p.append(out) + continue + else: + if base64buffer != 0: + i += 1 + errmsg = "non-zero padding bits in shift sequence" + out, i = unicode_call_errorhandler( + errors, "utf7", errmsg, s, startinpos, i + ) + p.append(out) + continue + if surrogate and _DECODE_DIRECT(ch): + p.append(chr(surrogate)) + surrogate = 0 + if ch == ord("-"): + i += 1 + elif ch == ord("+"): startinpos = i i += 1 - if (i= 6 or (base64bits > 0 and base64buffer != 0): + errmsg = "unterminated shift sequence" + out, i = unicode_call_errorhandler( + errors, "utf7", errmsg, s, startinpos, size + ) + p.append(out) + + return p, size + + +def _ENCODE_DIRECT(ch, encodeSetO, encodeWhiteSpace): + c = ord(ch) if isinstance(ch, str) else ch + if c > 127: + return False + if utf7_special[c] == 0: + return True + if utf7_special[c] == 2: + return not encodeWhiteSpace + if utf7_special[c] == 3: + return not encodeSetO + return False - if (inShift) : - #XXX This aint right - endinpos = size - raise UnicodeDecodeError("unterminated shift sequence") - - return p def PyUnicode_EncodeUTF7(s, size, encodeSetO, encodeWhiteSpace, errors): - -# /* It might be possible to tighten this worst case */ inShift = False - i = 0 - bitsleft = 0 - charsleft = 0 + base64bits = 0 + base64buffer = 0 out = [] - for ch in s: - if (not inShift) : - if (ch == '+'): - out.append(b'+-') - elif (SPECIAL(ch, encodeSetO, encodeWhiteSpace)): - charsleft = ord(ch) - bitsleft = 16 - out.append(b'+') - p, bitsleft = ENCODE( charsleft, bitsleft) - out.append(p) - inShift = bitsleft > 0 + + for i, ch in enumerate(s): + ch_ord = ord(ch) + if inShift: + if _ENCODE_DIRECT(ch, encodeSetO, encodeWhiteSpace): + # shifting out + if base64bits: + out.append(B64(base64buffer << (6 - base64bits))) + base64buffer = 0 + base64bits = 0 + inShift = False + if B64CHAR(ch) or ch == "-": + out.append(b"-") + out.append(bytes([ch_ord])) else: - out.append(bytes([ord(ch)])) + # encode character in base64 + if ch_ord >= 0x10000: + # split into surrogate pair + hi = 0xD800 | ((ch_ord - 0x10000) >> 10) + lo = 0xDC00 | ((ch_ord - 0x10000) & 0x3FF) + base64bits += 16 + base64buffer = (base64buffer << 16) | hi + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 + ch_ord = lo + + base64bits += 16 + base64buffer = (base64buffer << 16) | ch_ord + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 else: - if (not SPECIAL(ch, encodeSetO, encodeWhiteSpace)): - out.append(B64((charsleft) << (6-bitsleft))) - charsleft = 0 - bitsleft = 0 -## /* Characters not in the BASE64 set implicitly unshift the sequence -## so no '-' is required, except if the character is itself a '-' */ - if (B64CHAR(ch) or ch == '-'): - out.append(b'-') - inShift = False - out.append(bytes([ord(ch)])) + if ch == "+": + out.append(b"+-") + elif _ENCODE_DIRECT(ch, encodeSetO, encodeWhiteSpace): + out.append(bytes([ch_ord])) else: - bitsleft += 16 - charsleft = (((charsleft) << 16) | ord(ch)) - p, bitsleft = ENCODE(charsleft, bitsleft) - out.append(p) -## /* If the next character is special then we dont' need to terminate -## the shift sequence. If the next character is not a BASE64 character -## or '-' then the shift sequence will be terminated implicitly and we -## don't have to insert a '-'. */ - - if (bitsleft == 0): - if (i + 1 < size): - ch2 = s[i+1] - - if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)): - pass - elif (B64CHAR(ch2) or ch2 == '-'): - out.append(b'-') - inShift = False - else: + out.append(b"+") + inShift = True + # encode character in base64 + if ch_ord >= 0x10000: + hi = 0xD800 | ((ch_ord - 0x10000) >> 10) + lo = 0xDC00 | ((ch_ord - 0x10000) & 0x3FF) + base64bits += 16 + base64buffer = (base64buffer << 16) | hi + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 + ch_ord = lo + + base64bits += 16 + base64buffer = (base64buffer << 16) | ch_ord + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 + + if base64bits == 0: + if i + 1 < size: + ch2 = s[i + 1] + if _ENCODE_DIRECT(ch2, encodeSetO, encodeWhiteSpace): + if B64CHAR(ch2) or ch2 == "-": + out.append(b"-") inShift = False else: - out.append(b'-') + out.append(b"-") inShift = False - i += 1 - - if (bitsleft): - out.append(B64(charsleft << (6-bitsleft) ) ) - out.append(b'-') + + if base64bits: + out.append(B64(base64buffer << (6 - base64bits))) + if inShift: + out.append(b"-") return out -unicode_empty = '' -def unicodeescape_string(s, size, quotes): +unicode_empty = "" + +def unicodeescape_string(s, size, quotes): p = [] - if (quotes) : - if (s.find('\'') != -1 and s.find('"') == -1): + if quotes: + if s.find("'") != -1 and s.find('"') == -1: p.append(b'"') else: - p.append(b'\'') + p.append(b"'") pos = 0 - while (pos < size): + while pos < size: ch = s[pos] - #/* Escape quotes */ - if (quotes and (ch == p[1] or ch == '\\')): - p.append(b'\\%c' % ord(ch)) + # /* Escape quotes */ + if quotes and (ch == p[1] or ch == "\\"): + p.append(b"\\%c" % ord(ch)) pos += 1 continue -#ifdef Py_UNICODE_WIDE - #/* Map 21-bit characters to '\U00xxxxxx' */ - elif (ord(ch) >= 0x10000): - p.append(b'\\U%08x' % ord(ch)) + # ifdef Py_UNICODE_WIDE + # /* Map 21-bit characters to '\U00xxxxxx' */ + elif ord(ch) >= 0x10000: + p.append(b"\\U%08x" % ord(ch)) pos += 1 - continue -#endif - #/* Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes */ - elif (ord(ch) >= 0xD800 and ord(ch) < 0xDC00): - pos += 1 - ch2 = s[pos] - - if (ord(ch2) >= 0xDC00 and ord(ch2) <= 0xDFFF): - ucs = (((ord(ch) & 0x03FF) << 10) | (ord(ch2) & 0x03FF)) + 0x00010000 - p.append(b'\\U%08x' % ucs) - pos += 1 - continue - - #/* Fall through: isolated surrogates are copied as-is */ - pos -= 1 - - #/* Map 16-bit characters to '\uxxxx' */ - if (ord(ch) >= 256): - p.append(b'\\u%04x' % ord(ch)) - - #/* Map special whitespace to '\t', \n', '\r' */ - elif (ch == '\t'): - p.append(b'\\t') - - elif (ch == '\n'): - p.append(b'\\n') - - elif (ch == '\r'): - p.append(b'\\r') - - elif (ch == '\\'): - p.append(b'\\\\') - - #/* Map non-printable US ASCII to '\xhh' */ - elif (ch < ' ' or ch >= chr(0x7F)) : - p.append(b'\\x%02x' % ord(ch)) - #/* Copy everything else as-is */ + continue + # endif + # /* Map 16-bit characters to '\uxxxx' */ + if ord(ch) >= 256: + p.append(b"\\u%04x" % ord(ch)) + + # /* Map special whitespace to '\t', \n', '\r' */ + elif ch == "\t": + p.append(b"\\t") + + elif ch == "\n": + p.append(b"\\n") + + elif ch == "\r": + p.append(b"\\r") + + elif ch == "\\": + p.append(b"\\\\") + + # /* Map non-printable US ASCII to '\xhh' */ + elif ch < " " or ch >= chr(0x7F): + p.append(b"\\x%02x" % ord(ch)) + # /* Copy everything else as-is */ else: p.append(bytes([ord(ch)])) pos += 1 - if (quotes): + if quotes: p.append(p[0]) return p -def PyUnicode_DecodeASCII(s, size, errors): -# /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 and ord(s) < 128) : +def PyUnicode_DecodeASCII(s, size, errors): + # /* ASCII is equivalent to the first 128 ordinals in Unicode. */ + if size == 1 and ord(s) < 128: return [chr(ord(s))] - if (size == 0): - return [''] #unicode('') + if size == 0: + return [""] # unicode('') p = [] pos = 0 while pos < len(s): @@ -661,54 +1164,50 @@ def PyUnicode_DecodeASCII(s, size, errors): p += chr(c) pos += 1 else: - res = unicode_call_errorhandler( - errors, "ascii", "ordinal not in range(128)", - s, pos, pos+1) + errors, "ascii", "ordinal not in range(128)", s, pos, pos + 1 + ) p += res[0] pos = res[1] return p -def PyUnicode_EncodeASCII(p, size, errors): +def PyUnicode_EncodeASCII(p, size, errors): return unicode_encode_ucs1(p, size, errors, 128) -def PyUnicode_AsASCIIString(unistr): +def PyUnicode_AsASCIIString(unistr): if not type(unistr) == str: raise TypeError - return PyUnicode_EncodeASCII(str(unistr), - len(str), - None) + return PyUnicode_EncodeASCII(unistr, len(unistr), None) -def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=True): - bo = 0 #/* assume native ordering by default */ +def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=True): + bo = 0 # /* assume native ordering by default */ consumed = 0 errmsg = "" - if sys.byteorder == 'little': + if sys.byteorder == "little": ihi = 1 ilo = 0 else: ihi = 0 ilo = 1 - - #/* Unpack UTF-16 encoded data */ + # /* Unpack UTF-16 encoded data */ -## /* Check for BOM marks (U+FEFF) in the input and adjust current -## byte order setting accordingly. In native mode, the leading BOM -## mark is skipped, in all other modes, it is copied to the output -## stream as-is (giving a ZWNBSP character). */ + ## /* Check for BOM marks (U+FEFF) in the input and adjust current + ## byte order setting accordingly. In native mode, the leading BOM + ## mark is skipped, in all other modes, it is copied to the output + ## stream as-is (giving a ZWNBSP character). */ q = 0 p = [] - if byteorder == 'native': - if (size >= 2): + if byteorder == "native": + if size >= 2: bom = (s[ihi] << 8) | s[ilo] -#ifdef BYTEORDER_IS_LITTLE_ENDIAN - if sys.byteorder == 'little': - if (bom == 0xFEFF): + # ifdef BYTEORDER_IS_LITTLE_ENDIAN + if sys.byteorder == "little": + if bom == 0xFEFF: q += 2 bo = -1 elif bom == 0xFFFE: @@ -721,118 +1220,140 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru elif bom == 0xFFFE: q += 2 bo = -1 - elif byteorder == 'little': + elif byteorder == "little": bo = -1 else: bo = 1 - - if (size == 0): - return [''], 0, bo - - if (bo == -1): - #/* force LE */ + + if size == 0: + return [""], 0, bo + + if bo == -1: + # /* force LE */ ihi = 1 ilo = 0 - elif (bo == 1): - #/* force BE */ + elif bo == 1: + # /* force BE */ ihi = 0 ilo = 1 - while (q < len(s)): - - #/* remaining bytes at the end? (size should be even) */ - if (len(s)-q<2): + # The order the data turned out to be in is the one an error names. + encoding = "utf-16-be" if ihi == 0 else "utf-16-le" + + while q < len(s): + # /* remaining bytes at the end? (size should be even) */ + if len(s) - q < 2: if not final: break - errmsg = "truncated data" - startinpos = q - endinpos = len(s) - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) -# /* The remaining input chars are ignored if the callback -## chooses to skip the input */ - - ch = (s[q+ihi] << 8) | s[q+ilo] - q += 2 - - if (ch < 0xD800 or ch > 0xDFFF): + res, q = unicode_call_errorhandler( + errors, encoding, "truncated data", s, q, len(s), True + ) + p.append(res) + break + + ch = (s[q + ihi] << 8) | s[q + ilo] + + if ch < 0xD800 or ch > 0xDFFF: p.append(chr(ch)) - continue - - #/* UTF-16 code pair: */ - if (q >= len(s)): - errmsg = "unexpected end of data" - startinpos = q-2 - endinpos = len(s) - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - if (0xD800 <= ch and ch <= 0xDBFF): - ch2 = (s[q+ihi] << 8) | s[q+ilo] q += 2 - if (0xDC00 <= ch2 and ch2 <= 0xDFFF): - #ifndef Py_UNICODE_WIDE - if sys.maxunicode < 65536: - p += [chr(ch), chr(ch2)] + continue + + # /* UTF-16 code pair: high surrogate */ + if 0xD800 <= ch <= 0xDBFF: + if q + 4 <= len(s): + ch2 = (s[q + 2 + ihi] << 8) | s[q + 2 + ilo] + if 0xDC00 <= ch2 <= 0xDFFF: + # Valid surrogate pair - always assemble + p.append(chr((((ch & 0x3FF) << 10) | (ch2 & 0x3FF)) + 0x10000)) + q += 4 + continue else: - p.append(chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000)) - #endif + # High surrogate followed by non-low-surrogate + if errors == "surrogatepass": + p.append(chr(ch)) + q += 2 + continue + res, q = unicode_call_errorhandler( + errors, encoding, "illegal UTF-16 surrogate", s, q, q + 2, True + ) + p.append(res) + else: + # High surrogate at end of data + if not final: + break + if errors == "surrogatepass": + p.append(chr(ch)) + q += 2 + continue + res, q = unicode_call_errorhandler( + errors, encoding, "unexpected end of data", s, q, len(s), True + ) + p.append(res) + else: + # Low surrogate without preceding high surrogate + if errors == "surrogatepass": + p.append(chr(ch)) + q += 2 continue + res, q = unicode_call_errorhandler( + errors, encoding, "illegal encoding", s, q, q + 2, True + ) + p.append(res) - else: - errmsg = "illegal UTF-16 surrogate" - startinpos = q-4 - endinpos = startinpos+2 - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - errmsg = "illegal encoding" - startinpos = q-2 - endinpos = startinpos+2 - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - return p, q, bo + # moved out of local scope, especially because it didn't # have any nested variables. + def STORECHAR(CH, byteorder): - hi = (CH >> 8) & 0xff - lo = CH & 0xff - if byteorder == 'little': + hi = (CH >> 8) & 0xFF + lo = CH & 0xFF + if byteorder == "little": return [lo, hi] else: return [hi, lo] -def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'): -# /* Offsets from p for storing byte pairs in the right order. */ +def PyUnicode_EncodeUTF16(s, size, errors, byteorder="little"): + # /* Offsets from p for storing byte pairs in the right order. */ - p = [] bom = sys.byteorder - if (byteorder == 'native'): - + if byteorder == "native": bom = sys.byteorder p += STORECHAR(0xFEFF, bom) - - if (size == 0): - return "" - if (byteorder == 'little' ): - bom = 'little' - elif (byteorder == 'big'): - bom = 'big' + if byteorder == "little": + bom = "little" + elif byteorder == "big": + bom = "big" + # Only the mode that writes a BOM leaves the order out of its name. + encoding = "utf-16" if byteorder == "native" else "utf-16-" + bom[0] + "e" - for c in s: - ch = ord(c) - ch2 = 0 - if (ch >= 0x10000) : - ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) - ch = 0xD800 | ((ch-0x10000) >> 10) + def store(cp): + if cp >= 0x10000: + cp -= 0x10000 + return STORECHAR(0xD800 | (cp >> 10), bom) + STORECHAR( + 0xDC00 | (cp & 0x3FF), bom + ) + return STORECHAR(cp, bom) - p += STORECHAR(ch, bom) - if (ch2): - p += STORECHAR(ch2, bom) + pos = 0 + while pos < len(s): + ch = ord(s[pos]) + if 0xD800 <= ch <= 0xDFFF and errors != "surrogatepass": + start = pos + res, pos = unicode_call_errorhandler( + errors, encoding, "surrogates not allowed", s, pos, pos + 1, False + ) + p += _encoded_replacement(res, store, 2, encoding, s, start) + else: + p += store(ch) + pos += 1 return p @@ -840,119 +1361,149 @@ def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'): def PyUnicode_DecodeMBCS(s, size, errors): pass + def PyUnicode_EncodeMBCS(p, size, errors): pass -def unicode_call_errorhandler(errors, encoding, - reason, input, startinpos, endinpos, decode=True): - + +def unicode_call_errorhandler( + errors, encoding, reason, input, startinpos, endinpos, decode=True +): errorHandler = lookup_error(errors) if decode: - exceptionObject = UnicodeDecodeError(encoding, input, startinpos, endinpos, reason) + exceptionObject = UnicodeDecodeError( + encoding, input, startinpos, endinpos, reason + ) else: - exceptionObject = UnicodeEncodeError(encoding, input, startinpos, endinpos, reason) + exceptionObject = UnicodeEncodeError( + encoding, input, startinpos, endinpos, reason + ) res = errorHandler(exceptionObject) - if isinstance(res, tuple) and isinstance(res[0], str) and isinstance(res[1], int): + if ( + isinstance(res, tuple) + and isinstance(res[0], (str, bytes)) + and isinstance(res[1], int) + ): newpos = res[1] - if (newpos < 0): + if newpos < 0: newpos = len(input) + newpos if newpos < 0 or newpos > len(input): - raise IndexError( "position %d from error handler out of bounds" % newpos) + raise IndexError("position %d from error handler out of bounds" % newpos) return res[0], newpos else: - raise TypeError("encoding error handler must return (unicode, int) tuple, not %s" % repr(res)) + raise TypeError( + "encoding error handler must return (unicode, int) tuple, not %s" + % repr(res) + ) + + +# /* --- Latin-1 Codec ------------------------------------------------------ */ -#/* --- Latin-1 Codec ------------------------------------------------------ */ def PyUnicode_DecodeLatin1(s, size, errors): - #/* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ -## if (size == 1): -## return [PyUnicode_FromUnicode(s, 1)] + # /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ + ## if (size == 1): + ## return [PyUnicode_FromUnicode(s, 1)] pos = 0 p = [] - while (pos < size): + while pos < size: p += chr(s[pos]) pos += 1 return p + def unicode_encode_ucs1(p, size, errors, limit): - if limit == 256: reason = "ordinal not in range(256)" encoding = "latin-1" else: reason = "ordinal not in range(128)" encoding = "ascii" - - if (size == 0): + + if size == 0: return [] res = bytearray() pos = 0 while pos < len(p): - #for ch in p: + # for ch in p: ch = p[pos] - + if ord(ch) < limit: res.append(ord(ch)) pos += 1 else: - #/* startpos for collecting unencodable chars */ - collstart = pos - collend = pos+1 + # /* startpos for collecting unencodable chars */ + collstart = pos + collend = pos + 1 while collend < len(p) and ord(p[collend]) >= limit: collend += 1 - x = unicode_call_errorhandler(errors, encoding, reason, p, collstart, collend, False) - res += x[0].encode() + x = unicode_call_errorhandler( + errors, encoding, reason, p, collstart, collend, False + ) + replacement = x[0] + if isinstance(replacement, bytes): + res += replacement + else: + res += replacement.encode() pos = x[1] - + return res + def PyUnicode_EncodeLatin1(p, size, errors): res = unicode_encode_ucs1(p, size, errors, 256) return res -hexdigits = [ord(hex(i)[-1]) for i in range(16)]+[ord(hex(i)[-1].upper()) for i in range(10, 16)] + +hexdigits = [ord(hex(i)[-1]) for i in range(16)] + [ + ord(hex(i)[-1].upper()) for i in range(10, 16) +] + def hex_number_end(s, pos, digits): target_end = pos + digits - while pos < target_end and pos < len(s) and s[pos] in hexdigits: + while pos < target_end and pos < len(s) and s[pos] in hexdigits: pos += 1 return pos + def hexescape(s, pos, digits, message, errors): ch = 0 p = [] number_end = hex_number_end(s, pos, digits) if number_end - pos != digits: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-2, number_end) + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 2, number_end + ) p.append(x[0]) pos = x[1] else: - ch = int(s[pos:pos+digits], 16) - #/* when we get here, ch is a 32-bit unicode character */ + ch = int(s[pos : pos + digits], 16) + # /* when we get here, ch is a 32-bit unicode character */ if ch <= sys.maxunicode: p.append(chr(ch)) pos += digits - elif (ch <= 0x10ffff): + elif ch <= 0x10FFFF: ch -= 0x10000 p.append(chr(0xD800 + (ch >> 10))) - p.append(chr(0xDC00 + (ch & 0x03FF))) + p.append(chr(0xDC00 + (ch & 0x03FF))) pos += digits else: message = "illegal Unicode character" - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-2, - pos+digits) + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 2, pos + digits + ) p.append(x[0]) pos = x[1] res = p return res, pos + def PyUnicode_DecodeUnicodeEscape(s, size, errors, final): + if size == 0: + return "", 0 - if (size == 0): - return '' - if isinstance(s, str): s = s.encode() @@ -960,131 +1511,168 @@ def PyUnicode_DecodeUnicodeEscape(s, size, errors, final): p = [] pos = 0 - while (pos < size): -## /* Non-escape characters are interpreted as Unicode ordinals */ - if (chr(s[pos]) != '\\') : + while pos < size: + ## /* Non-escape characters are interpreted as Unicode ordinals */ + if s[pos] != ord("\\"): p.append(chr(s[pos])) pos += 1 continue -## /* \ - Escapes */ - else: - pos += 1 - if pos >= len(s): - errmessage = "\\ at end of string" - unicode_call_errorhandler(errors, "unicodeescape", errmessage, s, pos-1, size) - ch = chr(s[pos]) - pos += 1 - ## /* \x escapes */ - if ch == '\n': pass - elif ch == '\\': p += '\\' - elif ch == '\'': p += '\'' - elif ch == '\"': p += '\"' - elif ch == 'b' : p += '\b' - elif ch == 'f' : p += '\014' #/* FF */ - elif ch == 't' : p += '\t' - elif ch == 'n' : p += '\n' - elif ch == 'r' : p += '\r' - elif ch == 'v' : p += '\013' #break; /* VT */ - elif ch == 'a' : p += '\007' # break; /* BEL, not classic C */ - elif '0' <= ch <= '7': - x = ord(ch) - ord('0') - if pos < size: - ch = chr(s[pos]) - if '0' <= ch <= '7': - pos += 1 - x = (x<<3) + ord(ch) - ord('0') - if pos < size: - ch = chr(s[pos]) - if '0' <= ch <= '7': - pos += 1 - x = (x<<3) + ord(ch) - ord('0') - p.append(chr(x)) - ## /* hex escapes */ - ## /* \xXX */ - elif ch == 'x': + ## /* \ - Escapes */ + escape_start = pos + pos += 1 + if pos >= size: + if not final: + pos = escape_start + break + errmessage = "\\ at end of string" + unicode_call_errorhandler( + errors, "unicodeescape", errmessage, s, pos - 1, size + ) + break + ch = chr(s[pos]) + pos += 1 + ## /* \x escapes */ + if ch == "\n": + pass + elif ch == "\\": + p += "\\" + elif ch == "'": + p += "'" + elif ch == '"': + p += '"' + elif ch == "b": + p += "\b" + elif ch == "f": + p += "\014" # /* FF */ + elif ch == "t": + p += "\t" + elif ch == "n": + p += "\n" + elif ch == "r": + p += "\r" + elif ch == "v": + p += "\013" # break; /* VT */ + elif ch == "a": + p += "\007" # break; /* BEL, not classic C */ + elif "0" <= ch <= "7": + x = ord(ch) - ord("0") + if pos < size: + ch = chr(s[pos]) + if "0" <= ch <= "7": + pos += 1 + x = (x << 3) + ord(ch) - ord("0") + if pos < size: + ch = chr(s[pos]) + if "0" <= ch <= "7": + pos += 1 + x = (x << 3) + ord(ch) - ord("0") + p.append(chr(x)) + ## /* hex escapes */ + ## /* \xXX */ + elif ch in ("x", "u", "U"): + if ch == "x": digits = 2 message = "truncated \\xXX escape" - x = hexescape(s, pos, digits, message, errors) - p += x[0] - pos = x[1] - - # /* \uXXXX */ - elif ch == 'u': + elif ch == "u": digits = 4 message = "truncated \\uXXXX escape" + else: + digits = 8 + message = "truncated \\UXXXXXXXX escape" + number_end = hex_number_end(s, pos, digits) + if number_end - pos != digits: + if not final: + pos = escape_start + break x = hexescape(s, pos, digits, message, errors) p += x[0] pos = x[1] - - # /* \UXXXXXXXX */ - elif ch == 'U': - digits = 8 - message = "truncated \\UXXXXXXXX escape" + else: x = hexescape(s, pos, digits, message, errors) p += x[0] pos = x[1] -## /* \N{name} */ - elif ch == 'N': - message = "malformed \\N character escape" - # pos += 1 - look = pos - try: - import unicodedata - except ImportError: - message = "\\N escapes not supported (can't load unicodedata module)" - unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, size) - if look < size and chr(s[look]) == '{': - #/* look for the closing brace */ - while (look < size and chr(s[look]) != '}'): - look += 1 - if (look > pos+1 and look < size and chr(s[look]) == '}'): - #/* found a name. look it up in the unicode database */ - message = "unknown Unicode character name" - st = s[pos+1:look] - try: - chr_codec = unicodedata.lookup("%s" % st) - except LookupError as e: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1) - else: - x = chr_codec, look + 1 - p.append(x[0]) - pos = x[1] - else: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1) - else: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1) + ## /* \N{name} */ + elif ch == "N": + message = "malformed \\N character escape" + look = pos + try: + import unicodedata + except ImportError: + message = "\\N escapes not supported (can't load unicodedata module)" + unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, size + ) + continue + if look < size and chr(s[look]) == "{": + # /* look for the closing brace */ + while look < size and chr(s[look]) != "}": + look += 1 + if look > pos + 1 and look < size and chr(s[look]) == "}": + # /* found a name. look it up in the unicode database */ + message = "unknown Unicode character name" + st = s[pos + 1 : look] + try: + chr_codec = unicodedata.lookup("%s" % st) + except LookupError as e: + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, look + 1 + ) + else: + x = chr_codec, look + 1 + p.append(x[0]) + pos = x[1] + else: + if not final: + pos = escape_start + break + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, look + 1 + ) + p.append(x[0]) + pos = x[1] else: - if not found_invalid_escape: - found_invalid_escape = True - warnings.warn("invalid escape sequence '\\%c'" % ch, DeprecationWarning, 2) - p.append('\\') - p.append(ch) - return p + if not final: + pos = escape_start + break + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, look + 1 + ) + p.append(x[0]) + pos = x[1] + else: + if not found_invalid_escape: + found_invalid_escape = True + warnings.warn( + "invalid escape sequence '\\%c'" % ch, DeprecationWarning, 2 + ) + p.append("\\") + p.append(ch) + return p, pos + def PyUnicode_EncodeRawUnicodeEscape(s, size): - - if (size == 0): - return b'' + if size == 0: + return b"" p = bytearray() for ch in s: -# /* Map 32-bit characters to '\Uxxxxxxxx' */ - if (ord(ch) >= 0x10000): - p += b'\\U%08x' % ord(ch) - elif (ord(ch) >= 256) : -# /* Map 16-bit characters to '\uxxxx' */ - p += b'\\u%04x' % (ord(ch)) -# /* Copy everything else as-is */ + # /* Map 32-bit characters to '\Uxxxxxxxx' */ + if ord(ch) >= 0x10000: + p += b"\\U%08x" % ord(ch) + elif ord(ch) >= 256: + # /* Map 16-bit characters to '\uxxxx' */ + p += b"\\u%04x" % (ord(ch)) + # /* Copy everything else as-is */ else: p.append(ord(ch)) - - #p += '\0' + + # p += '\0' return p -def charmapencode_output(c, mapping): +def charmapencode_output(c, mapping): rep = mapping[c] - if isinstance(rep, int) or isinstance(rep, int): + if isinstance(rep, int): if rep < 256: return [rep] else: @@ -1098,144 +1686,171 @@ def charmapencode_output(c, mapping): else: raise TypeError("character mapping must return integer, None or str") -def PyUnicode_EncodeCharmap(p, size, mapping='latin-1', errors='strict'): -## /* the following variable is used for caching string comparisons -## * -1=not initialized, 0=unknown, 1=strict, 2=replace, -## * 3=ignore, 4=xmlcharrefreplace */ +def charmap_encoding_error(p, size, inpos, mapping, errors, res): + # /* find all unencodable characters */ + collendpos = inpos + 1 + while collendpos < size: + try: + charmapencode_output(ord(p[collendpos]), mapping) + except KeyError: + collendpos += 1 + else: + break + + replacement, newpos = unicode_call_errorhandler( + errors, + "charmap", + "character maps to ", + p, + inpos, + collendpos, + False, + ) + if isinstance(replacement, bytes): + res += list(replacement) + else: + for y in replacement: + try: + res += charmapencode_output(ord(y), mapping) + except KeyError: + raise UnicodeEncodeError( + "charmap", p, inpos, collendpos, "character maps to " + ) + return newpos + + +def PyUnicode_EncodeCharmap(p, size, mapping="latin-1", errors="strict"): + ## /* the following variable is used for caching string comparisons + ## * -1=not initialized, 0=unknown, 1=strict, 2=replace, + ## * 3=ignore, 4=xmlcharrefreplace */ -# /* Default to Latin-1 */ - if mapping == 'latin-1': + # /* Default to Latin-1 */ + if mapping == "latin-1": return PyUnicode_EncodeLatin1(p, size, errors) - if (size == 0): - return b'' + if size == 0: + return b"" inpos = 0 res = [] - while (inpos", p, inpos, inpos+1, False) - try: - for y in x[0]: - res += charmapencode_output(ord(y), mapping) - except KeyError: - raise UnicodeEncodeError("charmap", p, inpos, inpos+1, - "character maps to ") - inpos += 1 + inpos = charmap_encoding_error(p, size, inpos, mapping, errors, res) + else: + res += x + inpos += 1 return res -def PyUnicode_DecodeCharmap(s, size, mapping, errors): -## /* Default to Latin-1 */ - if (mapping == None): +def PyUnicode_DecodeCharmap(s, size, mapping, errors): + ## /* Default to Latin-1 */ + if mapping == None: return PyUnicode_DecodeLatin1(s, size, errors) - if (size == 0): - return '' + if size == 0: + return "" p = [] inpos = 0 - while (inpos< len(s)): - - #/* Get mapping (char ordinal -> integer, Unicode char or None) */ + while inpos < len(s): + # /* Get mapping (char ordinal -> integer, Unicode char or None) */ ch = s[inpos] try: x = mapping[ch] if isinstance(x, int): - if x < 65536: + if x == 0xFFFE: + raise KeyError + if 0 <= x <= 0x10FFFF: p += chr(x) else: - raise TypeError("character mapping must be in range(65536)") + raise TypeError( + "character mapping must be in range(0x%x)" % (0x110000,) + ) elif isinstance(x, str): + if len(x) == 1 and x == "\ufffe": + raise KeyError p += x - elif not x: + elif x is None: raise KeyError else: raise TypeError - except KeyError: - x = unicode_call_errorhandler(errors, "charmap", - "character maps to ", s, inpos, inpos+1) + except (KeyError, IndexError): + x = unicode_call_errorhandler( + errors, "charmap", "character maps to ", s, inpos, inpos + 1 + ) p += x[0] inpos += 1 return p -def PyUnicode_DecodeRawUnicodeEscape(s, size, errors, final): - if (size == 0): - return '' +def PyUnicode_DecodeRawUnicodeEscape(s, size, errors, final): + if size == 0: + return "", 0 if isinstance(s, str): s = s.encode() pos = 0 p = [] - while (pos < len(s)): - ch = chr(s[pos]) - #/* Non-escape characters are interpreted as Unicode ordinals */ - if (ch != '\\'): - p.append(ch) + while pos < len(s): + # /* Non-escape characters are interpreted as Unicode ordinals */ + if s[pos] != ord("\\"): + p.append(chr(s[pos])) pos += 1 - continue + continue startinpos = pos -## /* \u-escapes are only interpreted iff the number of leading -## backslashes is odd */ + p_len_before = len(p) + ## /* \u-escapes are only interpreted iff the number of leading + ## backslashes is odd */ bs = pos while pos < size: - if (s[pos] != ord('\\')): + if s[pos] != ord("\\"): break p.append(chr(s[pos])) pos += 1 - - if (pos >= size): + + if pos >= size: + if not final: + del p[p_len_before:] + pos = startinpos break - if (((pos - bs) & 1) == 0 or - (s[pos] != ord('u') and s[pos] != ord('U'))) : + if ((pos - bs) & 1) == 0 or (s[pos] != ord("u") and s[pos] != ord("U")): p.append(chr(s[pos])) pos += 1 continue - + p.pop(-1) - if s[pos] == ord('u'): - count = 4 - else: - count = 8 + count = 4 if s[pos] == ord("u") else 8 pos += 1 - #/* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */ + # /* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */ number_end = hex_number_end(s, pos, count) if number_end - pos != count: + if not final: + del p[p_len_before:] + pos = startinpos + break res = unicode_call_errorhandler( - errors, "rawunicodeescape", "truncated \\uXXXX", - s, pos-2, number_end) + errors, "rawunicodeescape", "truncated \\uXXXX", s, pos - 2, number_end + ) p.append(res[0]) pos = res[1] else: - x = int(s[pos:pos+count], 16) - #ifndef Py_UNICODE_WIDE - if sys.maxunicode > 0xffff: - if (x > sys.maxunicode): - res = unicode_call_errorhandler( - errors, "rawunicodeescape", "\\Uxxxxxxxx out of range", - s, pos-2, pos+count) - pos = res[1] - p.append(res[0]) - else: - p.append(chr(x)) - pos += count + x = int(s[pos : pos + count], 16) + if x > sys.maxunicode: + res = unicode_call_errorhandler( + errors, + "rawunicodeescape", + "\\Uxxxxxxxx out of range", + s, + pos - 2, + pos + count, + ) + pos = res[1] + p.append(res[0]) else: - if (x > 0x10000): - res = unicode_call_errorhandler( - errors, "rawunicodeescape", "\\Uxxxxxxxx out of range", - s, pos-2, pos+count) - pos = res[1] - p.append(res[0]) - - #endif - else: - p.append(chr(x)) - pos += count + p.append(chr(x)) + pos += count - return p + return p, pos diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index 38e1f764f00..0f95a575084 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1,12 +1,10 @@ -"""Concrete date/time and related types. - -See http://www.iana.org/time-zones/repository/tz-link.html for -time zone and DST data sources. -""" +"""Pure Python implementation of the datetime module.""" __all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", "MINYEAR", "MAXYEAR", "UTC") +__name__ = "datetime" + import time as _time import math as _math @@ -18,10 +16,10 @@ def _cmp(x, y): def _get_class_module(self): module_name = self.__class__.__module__ - if module_name == '_pydatetime': - return 'datetime' + if module_name == 'datetime': + return 'datetime.' else: - return module_name + return '' MINYEAR = 1 MAXYEAR = 9999 @@ -57,21 +55,21 @@ def _days_before_year(year): def _days_in_month(year, month): "year, month -> number of days in that month in that year." - assert 1 <= month <= 12, month + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" if month == 2 and _is_leap(year): return 29 return _DAYS_IN_MONTH[month] def _days_before_month(year, month): "year, month -> number of days in year preceding first day of month." - assert 1 <= month <= 12, 'month must be in 1..12' + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" return _DAYS_BEFORE_MONTH[month] + (month > 2 and _is_leap(year)) def _ymd2ord(year, month, day): "year, month, day -> ordinal, considering 01-Jan-0001 as day 1." - assert 1 <= month <= 12, 'month must be in 1..12' + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" dim = _days_in_month(year, month) - assert 1 <= day <= dim, ('day must be in 1..%d' % dim) + assert 1 <= day <= dim, f"day must be in 1..{dim}, not {day}" return (_days_before_year(year) + _days_before_month(year, month) + day) @@ -204,6 +202,17 @@ def _format_offset(off, sep=':'): s += '.%06d' % ss.microseconds return s +_normalize_century = None +def _need_normalize_century(): + global _normalize_century + if _normalize_century is None: + try: + _normalize_century = ( + _time.strftime("%Y", (99, 1, 1, 0, 0, 0, 0, 1, 0)) != "0099") + except ValueError: + _normalize_century = True + return _normalize_century + # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): # Don't call utcoffset() or tzname() unless actually needed. @@ -261,6 +270,20 @@ def _wrap_strftime(object, format, timetuple): # strftime is going to have at this: escape % Zreplace = s.replace('%', '%%') newformat.append(Zreplace) + # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so + # year 1000 for %G can go on the fast path. + elif (ch in 'YGFC' and timetuple[0] < 1000 and + _need_normalize_century()): + if ch == 'G': + year = int(_time.strftime("%G", timetuple)) + else: + year = timetuple[0] + if ch == 'C': + push('{:02}'.format(year // 100)) + else: + push('{:04}'.format(year)) + if ch == 'F': + push('-{:02}-{:02}'.format(*timetuple[1:3])) else: push('%') push(ch) @@ -335,7 +358,8 @@ def _find_isoformat_datetime_separator(dtstr): def _parse_isoformat_date(dtstr): # It is assumed that this is an ASCII-only string of lengths 7, 8 or 10, # see the comment on Modules/_datetimemodule.c:_find_isoformat_datetime_separator - assert len(dtstr) in (7, 8, 10) + if len(dtstr) not in (7, 8, 10): + raise ValueError("Invalid isoformat string") year = int(dtstr[0:4]) has_sep = dtstr[4] == '-' @@ -399,7 +423,7 @@ def _parse_hh_mm_ss_ff(tstr): if pos < len_str: if tstr[pos] not in '.,': - raise ValueError("Invalid microsecond component") + raise ValueError("Invalid microsecond separator") else: pos += 1 if not all(map(_is_ascii_digit, tstr[pos:])): @@ -430,6 +454,17 @@ def _parse_isoformat_time(tstr): time_comps = _parse_hh_mm_ss_ff(timestr) + hour, minute, second, microsecond = time_comps + became_next_day = False + error_from_components = False + if (hour == 24): + if all(time_comp == 0 for time_comp in time_comps[1:]): + hour = 0 + time_comps[0] = hour + became_next_day = True + else: + error_from_components = True + tzi = None if tz_pos == len_str and tstr[-1] == 'Z': tzi = timezone.utc @@ -445,7 +480,7 @@ def _parse_isoformat_time(tstr): # HH:MM:SS len: 8 # HH:MM:SS.f+ len: 10+ - if len(tzstr) in (0, 1, 3): + if len(tzstr) in (0, 1, 3) or tstr[tz_pos-1] == 'Z': raise ValueError("Malformed time zone string") tz_comps = _parse_hh_mm_ss_ff(tzstr) @@ -462,13 +497,13 @@ def _parse_isoformat_time(tstr): time_comps.append(tzi) - return time_comps + return time_comps, became_next_day, error_from_components # tuple[int, int, int] -> tuple[int, int, int] version of date.fromisocalendar def _isoweek_to_gregorian(year, week, day): # Year is bounded this way because 9999-12-31 is (9999, 52, 5) if not MINYEAR <= year <= MAXYEAR: - raise ValueError(f"Year is out of range: {year}") + raise ValueError(f"year must be in {MINYEAR}..{MAXYEAR}, not {year}") if not 0 < week < 53: out_of_range = True @@ -501,7 +536,7 @@ def _isoweek_to_gregorian(year, week, day): def _check_tzname(name): if name is not None and not isinstance(name, str): raise TypeError("tzinfo.tzname() must return None or string, " - "not '%s'" % type(name)) + f"not {type(name).__name__!r}") # name is the offset-producing method, "utcoffset" or "dst". # offset is what it returned. @@ -514,24 +549,24 @@ def _check_utc_offset(name, offset): if offset is None: return if not isinstance(offset, timedelta): - raise TypeError("tzinfo.%s() must return None " - "or timedelta, not '%s'" % (name, type(offset))) + raise TypeError(f"tzinfo.{name}() must return None " + f"or timedelta, not {type(offset).__name__!r}") if not -timedelta(1) < offset < timedelta(1): - raise ValueError("%s()=%s, must be strictly between " - "-timedelta(hours=24) and timedelta(hours=24)" % - (name, offset)) + raise ValueError("offset must be a timedelta " + "strictly between -timedelta(hours=24) and " + f"timedelta(hours=24), not {offset!r}") def _check_date_fields(year, month, day): year = _index(year) month = _index(month) day = _index(day) if not MINYEAR <= year <= MAXYEAR: - raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) + raise ValueError(f"year must be in {MINYEAR}..{MAXYEAR}, not {year}") if not 1 <= month <= 12: - raise ValueError('month must be in 1..12', month) + raise ValueError(f"month must be in 1..12, not {month}") dim = _days_in_month(year, month) if not 1 <= day <= dim: - raise ValueError('day must be in 1..%d' % dim, day) + raise ValueError(f"day {day} must be in range 1..{dim} for month {month} in year {year}") return year, month, day def _check_time_fields(hour, minute, second, microsecond, fold): @@ -540,20 +575,23 @@ def _check_time_fields(hour, minute, second, microsecond, fold): second = _index(second) microsecond = _index(microsecond) if not 0 <= hour <= 23: - raise ValueError('hour must be in 0..23', hour) + raise ValueError(f"hour must be in 0..23, not {hour}") if not 0 <= minute <= 59: - raise ValueError('minute must be in 0..59', minute) + raise ValueError(f"minute must be in 0..59, not {minute}") if not 0 <= second <= 59: - raise ValueError('second must be in 0..59', second) + raise ValueError(f"second must be in 0..59, not {second}") if not 0 <= microsecond <= 999999: - raise ValueError('microsecond must be in 0..999999', microsecond) + raise ValueError(f"microsecond must be in 0..999999, not {microsecond}") if fold not in (0, 1): - raise ValueError('fold must be either 0 or 1', fold) + raise ValueError(f"fold must be either 0 or 1, not {fold}") return hour, minute, second, microsecond, fold def _check_tzinfo_arg(tz): if tz is not None and not isinstance(tz, tzinfo): - raise TypeError("tzinfo argument must be None or of a tzinfo subclass") + raise TypeError( + "tzinfo argument must be None or of a tzinfo subclass, " + f"not {type(tz).__name__!r}" + ) def _divide_and_round(a, b): """divide a by b and round result to the nearest integer @@ -607,7 +645,19 @@ def __new__(cls, days=0, seconds=0, microseconds=0, # guide the C implementation; it's way more convoluted than speed- # ignoring auto-overflow-to-long idiomatic Python could be. - # XXX Check that all inputs are ints or floats. + for name, value in ( + ("days", days), + ("seconds", seconds), + ("microseconds", microseconds), + ("milliseconds", milliseconds), + ("minutes", minutes), + ("hours", hours), + ("weeks", weeks) + ): + if not isinstance(value, (int, float)): + raise TypeError( + f"unsupported type for timedelta {name} component: {type(value).__name__}" + ) # Final values, all integer. # s and us fit in 32-bit signed ints; d isn't bounded. @@ -708,9 +758,9 @@ def __repr__(self): args.append("microseconds=%d" % self._microseconds) if not args: args.append('0') - return "%s.%s(%s)" % (_get_class_module(self), - self.__class__.__qualname__, - ', '.join(args)) + return "%s%s(%s)" % (_get_class_module(self), + self.__class__.__qualname__, + ', '.join(args)) def __str__(self): mm, ss = divmod(self._seconds, 60) @@ -907,6 +957,7 @@ class date: fromtimestamp() today() fromordinal() + strptime() Operators: @@ -989,8 +1040,12 @@ def fromordinal(cls, n): @classmethod def fromisoformat(cls, date_string): """Construct a date from a string in ISO 8601 format.""" + if not isinstance(date_string, str): - raise TypeError('fromisoformat: argument must be str') + raise TypeError('Argument must be a str') + + if not date_string.isascii(): + raise ValueError('Argument must be an ASCII str') if len(date_string) not in (7, 8, 10): raise ValueError(f'Invalid isoformat string: {date_string!r}') @@ -1007,6 +1062,12 @@ def fromisocalendar(cls, year, week, day): This is the inverse of the date.isocalendar() function""" return cls(*_isoweek_to_gregorian(year, week, day)) + @classmethod + def strptime(cls, date_string, format): + """Parse a date string according to the given format (like time.strptime()).""" + import _strptime + return _strptime._strptime_datetime_date(cls, date_string, format) + # Conversions to string def __repr__(self): @@ -1016,11 +1077,11 @@ def __repr__(self): >>> repr(d) 'datetime.date(2010, 1, 1)' """ - return "%s.%s(%d, %d, %d)" % (_get_class_module(self), - self.__class__.__qualname__, - self._year, - self._month, - self._day) + return "%s%s(%d, %d, %d)" % (_get_class_module(self), + self.__class__.__qualname__, + self._year, + self._month, + self._day) # XXX These shouldn't depend on time.localtime(), because that # clips the usable dates to [1970 .. 2038). At least ctime() is # easily done without using strftime() -- that's better too because @@ -1327,6 +1388,7 @@ class time: Constructors: __new__() + strptime() Operators: @@ -1385,6 +1447,12 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold self._fold = fold return self + @classmethod + def strptime(cls, date_string, format): + """string, format -> new time parsed from a string (like time.strptime()).""" + import _strptime + return _strptime._strptime_datetime_time(cls, date_string, format) + # Read-only field accessors @property def hour(self): @@ -1513,7 +1581,7 @@ def __repr__(self): s = ", %d" % self._second else: s = "" - s= "%s.%s(%d, %d%s)" % (_get_class_module(self), + s = "%s%s(%d, %d%s)" % (_get_class_module(self), self.__class__.__qualname__, self._hour, self._minute, s) if self._tzinfo is not None: @@ -1555,7 +1623,7 @@ def fromisoformat(cls, time_string): time_string = time_string.removeprefix('T') try: - return cls(*_parse_isoformat_time(time_string)) + return cls(*_parse_isoformat_time(time_string)[0]) except Exception: raise ValueError(f'Invalid isoformat string: {time_string!r}') @@ -1869,10 +1937,27 @@ def fromisoformat(cls, date_string): if tstr: try: - time_components = _parse_isoformat_time(tstr) + time_components, became_next_day, error_from_components = _parse_isoformat_time(tstr) except ValueError: raise ValueError( f'Invalid isoformat string: {date_string!r}') from None + else: + if error_from_components: + raise ValueError("minute, second, and microsecond must be 0 when hour is 24") + + if became_next_day: + year, month, day = date_components + # Only wrap day/month when it was previously valid + if 1 <= month <= 12 and day <= (days_in_month := _days_in_month(year, month)): + # Calculate midnight of the next day + day += 1 + if day > days_in_month: + day = 1 + month += 1 + if month > 12: + month = 1 + year += 1 + date_components = [year, month, day] else: time_components = [0, 0, 0, 0, None] @@ -2072,9 +2157,9 @@ def __repr__(self): del L[-1] if L[-1] == 0: del L[-1] - s = "%s.%s(%s)" % (_get_class_module(self), - self.__class__.__qualname__, - ", ".join(map(str, L))) + s = "%s%s(%s)" % (_get_class_module(self), + self.__class__.__qualname__, + ", ".join(map(str, L))) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" @@ -2091,7 +2176,7 @@ def __str__(self): def strptime(cls, date_string, format): 'string, format -> new datetime parsed from a string (like time.strptime()).' import _strptime - return _strptime._strptime_datetime(cls, date_string, format) + return _strptime._strptime_datetime_datetime(cls, date_string, format) def utcoffset(self): """Return the timezone offset as timedelta positive east of UTC (negative west of @@ -2331,7 +2416,7 @@ def __new__(cls, offset, name=_Omitted): if not cls._minoffset <= offset <= cls._maxoffset: raise ValueError("offset must be a timedelta " "strictly between -timedelta(hours=24) and " - "timedelta(hours=24).") + f"timedelta(hours=24), not {offset!r}") return cls._create(offset, name) def __init_subclass__(cls): @@ -2371,12 +2456,12 @@ def __repr__(self): if self is self.utc: return 'datetime.timezone.utc' if self._name is None: - return "%s.%s(%r)" % (_get_class_module(self), - self.__class__.__qualname__, - self._offset) - return "%s.%s(%r, %r)" % (_get_class_module(self), - self.__class__.__qualname__, - self._offset, self._name) + return "%s%s(%r)" % (_get_class_module(self), + self.__class__.__qualname__, + self._offset) + return "%s%s(%r, %r)" % (_get_class_module(self), + self.__class__.__qualname__, + self._offset, self._name) def __str__(self): return self.tzname(None) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 97a629fe92c..2277578df54 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -104,8 +104,8 @@ class DecimalException(ArithmeticError): anything, though. handle -- Called when context._raise_error is called and the - trap_enabler is not set. First argument is self, second is the - context. More arguments can be given, those being after + trap_enabler is not set. First argument is self, second is + the context. More arguments can be given, those being after the explanation in _raise_error (For example, context._raise_error(NewError, '(-x)!', self._sign) would call NewError().handle(context, self._sign).) @@ -222,11 +222,12 @@ class InvalidContext(InvalidOperation): """Invalid context. Unknown rounding, for example. This occurs and signals invalid-operation if an invalid context was - detected during an operation. This can occur if contexts are not checked - on creation and either the precision exceeds the capability of the - underlying concrete representation or an unknown or unsupported rounding - was specified. These aspects of the context need only be checked when - the values are required to be used. The result is [0,qNaN]. + detected during an operation. This can occur if contexts are not + checked on creation and either the precision exceeds the capability of + the underlying concrete representation or an unknown or unsupported + rounding was specified. These aspects of the context need only be + checked when the values are required to be used. The result is + [0,qNaN]. """ def handle(self, context, *args): @@ -319,8 +320,9 @@ class FloatOperation(DecimalException, TypeError): Decimal.from_float() or context.create_decimal_from_float() do not set the flag. - Otherwise (the signal is trapped), only equality comparisons and explicit - conversions are silent. All other mixed operations raise FloatOperation. + Otherwise (the signal is trapped), only equality comparisons and + explicit conversions are silent. All other mixed operations raise + FloatOperation. """ # List of public traps and flags @@ -2898,8 +2900,8 @@ def compare_total(self, other, context=None): """Compares self to other using the abstract representations. This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. + value. Note that a total ordering is defined for all possible + abstract representations. """ other = _convert_other(other, raiseit=True) @@ -2970,7 +2972,8 @@ def compare_total(self, other, context=None): def compare_total_mag(self, other, context=None): """Compares self to other using abstract repr., ignoring sign. - Like compare_total, but with operand's sign ignored and assumed to be 0. + Like compare_total, but with operand's sign ignored and assumed to + be 0. """ other = _convert_other(other, raiseit=True) @@ -4107,9 +4110,9 @@ def create_decimal_from_float(self, f): def abs(self, a): """Returns the absolute value of the operand. - If the operand is negative, the result is the same as using the minus - operation on the operand. Otherwise, the result is the same as using - the plus operation on the operand. + If the operand is negative, the result is the same as using the + minus operation on the operand. Otherwise, the result is the same + as using the plus operation on the operand. >>> ExtendedContext.abs(Decimal('2.1')) Decimal('2.1') @@ -4165,16 +4168,17 @@ def canonical(self, a): def compare(self, a, b): """Compares values numerically. - If the signs of the operands differ, a value representing each operand - ('-1' if the operand is less than zero, '0' if the operand is zero or - negative zero, or '1' if the operand is greater than zero) is used in - place of that operand for the comparison instead of the actual - operand. + If the signs of the operands differ, a value representing each + operand ('-1' if the operand is less than zero, '0' if the operand + is zero or negative zero, or '1' if the operand is greater than + zero) is used in place of that operand for the comparison instead of + the actual operand. - The comparison is then effected by subtracting the second operand from - the first and then returning a value according to the result of the - subtraction: '-1' if the result is less than zero, '0' if the result is - zero or negative zero, or '1' if the result is greater than zero. + The comparison is then effected by subtracting the second operand + from the first and then returning a value according to the result of + the subtraction: '-1' if the result is less than zero, '0' if the + result is zero or negative zero, or '1' if the result is greater + than zero. >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3')) Decimal('-1') @@ -4237,8 +4241,8 @@ def compare_total(self, a, b): """Compares two operands using their abstract representation. This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. + value. Note that a total ordering is defined for all possible + abstract representations. >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9')) Decimal('-1') @@ -4265,7 +4269,8 @@ def compare_total(self, a, b): def compare_total_mag(self, a, b): """Compares two operands using their abstract representation ignoring sign. - Like compare_total, but with operand's sign ignored and assumed to be 0. + Like compare_total, but with operand's sign ignored and assumed to + be 0. """ a = _convert_other(a, raiseit=True) return a.compare_total_mag(b) @@ -4923,8 +4928,8 @@ def multiply(self, a, b): If either operand is a special value then the general rules apply. Otherwise, the operands are multiplied together - ('long multiplication'), resulting in a number which may be as long as - the sum of the lengths of the two operands. + ('long multiplication'), resulting in a number which may be as long + as the sum of the lengths of the two operands. >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3')) Decimal('3.60') @@ -5200,19 +5205,19 @@ def quantize(self, a, b): """Returns a value equal to 'a' (rounded), having the exponent of 'b'. The coefficient of the result is derived from that of the left-hand - operand. It may be rounded using the current rounding setting (if the - exponent is being increased), multiplied by a positive power of ten (if - the exponent is being decreased), or is unchanged (if the exponent is - already equal to that of the right-hand operand). + operand. It may be rounded using the current rounding setting (if + the exponent is being increased), multiplied by a positive power of + ten (if the exponent is being decreased), or is unchanged (if the + exponent is already equal to that of the right-hand operand). Unlike other operations, if the length of the coefficient after the quantize operation would be greater than precision then an Invalid - operation condition is raised. This guarantees that, unless there is - an error condition, the exponent of the result of a quantize is always - equal to that of the right-hand operand. + operation condition is raised. This guarantees that, unless there + is an error condition, the exponent of the result of a quantize is + always equal to that of the right-hand operand. - Also unlike other operations, quantize will never raise Underflow, even - if the result is subnormal and inexact. + Also unlike other operations, quantize will never raise Underflow, + even if the result is subnormal and inexact. >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001')) Decimal('2.170') @@ -5266,13 +5271,13 @@ def remainder(self, a, b): """Returns the remainder from integer division. The result is the residue of the dividend after the operation of - calculating integer division as described for divide-integer, rounded - to precision digits if necessary. The sign of the result, if - non-zero, is the same as that of the original dividend. + calculating integer division as described for divide-integer, + rounded to precision digits if necessary. The sign of the result, + if non-zero, is the same as that of the original dividend. - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). + This operation will fail under the same conditions as integer + division (that is, if integer division on the same two operands + would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3')) Decimal('2.1') @@ -5306,9 +5311,9 @@ def remainder_near(self, a, b): is chosen). If the result is equal to 0 then its sign will be the sign of a. - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). + This operation will fail under the same conditions as integer + division (that is, if integer division on the same two operands + would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3')) Decimal('-0.9') @@ -5366,8 +5371,8 @@ def rotate(self, a, b): def same_quantum(self, a, b): """Returns True if the two operands have the same exponent. - The result is never affected by either the sign or the coefficient of - either operand. + The result is never affected by either the sign or the coefficient + of either operand. >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001')) False @@ -5439,8 +5444,8 @@ def shift(self, a, b): def sqrt(self, a): """Square root of a non-negative number to context precision. - If the result must be inexact, it is rounded using the round-half-even - algorithm. + If the result must be inexact, it is rounded using the + round-half-even algorithm. >>> ExtendedContext.sqrt(Decimal('0')) Decimal('0') diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 48c8f770f81..cc336d29394 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -16,15 +16,16 @@ _setmode = None import io -from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) +from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END, Reader, Writer) # noqa: F401 valid_seek_flags = {0, 1, 2} # Hardwired values if hasattr(os, 'SEEK_HOLE') : valid_seek_flags.add(os.SEEK_HOLE) valid_seek_flags.add(os.SEEK_DATA) -# open() uses st_blksize whenever we can -DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes +# open() uses max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) +# when the device block size is available. +DEFAULT_BUFFER_SIZE = 128 * 1024 # bytes # NOTE: Base classes defined here are registered with the "official" ABCs # defined in io.py. We don't use real inheritance though, because we don't want @@ -82,27 +83,28 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, wrapped. (If a file descriptor is given, it is closed when the returned I/O object is closed, unless closefd is set to False.) - mode is an optional string that specifies the mode in which the file is - opened. It defaults to 'r' which means open for reading in text mode. Other - common values are 'w' for writing (truncating the file if it already - exists), 'x' for exclusive creation of a new file, and 'a' for appending - (which on some Unix systems, means that all writes append to the end of the - file regardless of the current seek position). In text mode, if encoding is - not specified the encoding used is platform dependent. (For reading and - writing raw bytes use binary mode and leave encoding unspecified.) The - available modes are: - - ========= =============================================================== + mode is an optional string that specifies the mode in which the file + is opened. It defaults to 'r' which means open for reading in text + mode. Other common values are 'w' for writing (truncating the file if + it already exists), 'x' for exclusive creation of a new file, and + 'a' for appending (which on some Unix systems, means that all writes + append to the end of the file regardless of the current seek position). + In text mode, if encoding is not specified the encoding used is platform + dependent. (For reading and writing raw bytes use binary mode and leave + encoding unspecified.) The available modes are: + + ========= ========================================================== Character Meaning - --------- --------------------------------------------------------------- + --------- ---------------------------------------------------------- 'r' open for reading (default) 'w' open for writing, truncating the file first 'x' create a new file and open it for writing - 'a' open for writing, appending to the end of the file if it exists + 'a' open for writing, appending to the end of the file if it + exists 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) - ========= =============================================================== + ========= ========================================================== The default mode is 'rt' (open for reading text). For binary random access, the mode 'w+b' opens and truncates the file to 0 bytes, while @@ -110,23 +112,23 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, raises an `FileExistsError` if the file already exists. Python distinguishes between files opened in binary and text modes, - even when the underlying operating system doesn't. Files opened in + even when the underlying operating system doesn't. Files opened in binary mode (appending 'b' to the mode argument) return contents as - bytes objects without any decoding. In text mode (the default, or when + bytes objects without any decoding. In text mode (the default, or when 't' is appended to the mode argument), the contents of the file are returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. buffering is an optional integer used to set the buffering policy. - Pass 0 to switch buffering off (only allowed in binary mode), 1 to select - line buffering (only usable in text mode), and an integer > 1 to indicate - the size of a fixed-size chunk buffer. When no buffering argument is - given, the default buffering policy works as follows: + Pass 0 to switch buffering off (only allowed in binary mode), 1 to + select line buffering (only usable in text mode), and an integer > 1 to + indicate the size of a fixed-size chunk buffer. When no buffering + argument is given, the default buffering policy works as follows: * Binary files are buffered in fixed-size chunks; the size of the buffer - is chosen using a heuristic trying to determine the underlying device's - "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`. - On many systems, the buffer will typically be 4096 or 8192 bytes long. + is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) when the device + block size is available. + On most systems, the buffer will typically be 128 kilobytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above @@ -146,8 +148,8 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, encoding error strings. newline is a string controlling how universal newlines works (it only - applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It works - as follows: + applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It + works as follows: * On input, if newline is None, universal newlines mode is enabled. Lines in the input can end in '\n', '\r', or '\r\n', and @@ -163,17 +165,17 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, other legal values, any '\n' characters written are translated to the given string. - closedfd is a bool. If closefd is False, the underlying file descriptor will - be kept open when the file is closed. This does not work when a file name is - given and must be True in that case. + closedfd is a bool. If closefd is False, the underlying file descriptor + will be kept open when the file is closed. This does not work when + a file name is given and must be True in that case. The newly created file is non-inheritable. A custom opener can be used by passing a callable as *opener*. The - underlying file descriptor for the file object is then obtained by calling - *opener* with (*file*, *flags*). *opener* must return an open file - descriptor (passing os.open as *opener* results in functionality similar to - passing None). + underlying file descriptor for the file object is then obtained by + calling *opener* with (*file*, *flags*). *opener* must return an open + file descriptor (passing os.open as *opener* results in functionality + similar to passing None). open() returns a file object whose type depends on the mode, and through which the standard file operations such as reading and writing @@ -238,18 +240,11 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, result = raw try: line_buffering = False - if buffering == 1 or buffering < 0 and raw.isatty(): + if buffering == 1 or buffering < 0 and raw._isatty_open_only(): buffering = -1 line_buffering = True if buffering < 0: - buffering = DEFAULT_BUFFER_SIZE - try: - bs = os.fstat(raw.fileno()).st_blksize - except (OSError, AttributeError): - pass - else: - if bs > 1: - buffering = bs + buffering = max(min(raw._blksize, 8192 * 1024), DEFAULT_BUFFER_SIZE) if buffering < 0: raise ValueError("invalid buffering size") if buffering == 0: @@ -357,10 +352,12 @@ def seek(self, pos, whence=0): interpreted relative to the position indicated by whence. Values for whence are ints: - * 0 -- start of stream (the default); offset should be zero or positive + * 0 -- start of stream (the default); offset should be zero or + positive * 1 -- current stream position; offset may be negative * 2 -- end of stream; offset is usually negative - Some operating systems / file systems could provide additional values. + Some operating systems / file systems could provide additional + values. Return an int indicating the new absolute position. """ @@ -373,8 +370,8 @@ def tell(self): def truncate(self, pos=None): """Truncate file to size bytes. - Size defaults to the current IO position as reported by tell(). Return - the new size. + Size defaults to the current IO position as reported by tell(). + Return the new size. """ self._unsupported("truncate") @@ -498,7 +495,8 @@ def __exit__(self, *args): def fileno(self): """Returns underlying file descriptor (an int) if one exists. - An OSError is raised if the IO object does not use a file descriptor. + An OSError is raised if the IO object does not use a file + descriptor. """ self._unsupported("fileno") @@ -941,22 +939,22 @@ def read1(self, size=-1): return self.read(size) def write(self, b): - if self.closed: - raise ValueError("write to closed file") if isinstance(b, str): raise TypeError("can't write str to binary stream") with memoryview(b) as view: + if self.closed: + raise ValueError("write to closed file") + n = view.nbytes # Size of any bytes-like object - if n == 0: - return 0 - pos = self._pos - if pos > len(self._buffer): - # Inserts null bytes between the current end of the file - # and the new write position. - padding = b'\x00' * (pos - len(self._buffer)) - self._buffer += padding - self._buffer[pos:pos + n] = b - self._pos += n + if n == 0: + return 0 + + pos = self._pos + if pos > len(self._buffer): + # Pad buffer to pos with null bytes. + self._buffer.resize(pos) + self._buffer[pos:pos + n] = view + self._pos += n return n def seek(self, pos, whence=0): @@ -1470,6 +1468,17 @@ def write(self, b): return BufferedWriter.write(self, b) +def _new_buffersize(bytes_read): + # Parallels _io/fileio.c new_buffersize + if bytes_read > 65536: + addend = bytes_read >> 3 + else: + addend = 256 + bytes_read + if addend < DEFAULT_BUFFER_SIZE: + addend = DEFAULT_BUFFER_SIZE + return bytes_read + addend + + class FileIO(RawIOBase): _fd = -1 _created = False @@ -1480,20 +1489,26 @@ class FileIO(RawIOBase): _closefd = True def __init__(self, file, mode='r', closefd=True, opener=None): - """Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading, - writing, exclusive creation or appending. The file will be created if it - doesn't exist when opened for writing or appending; it will be truncated - when opened for writing. A FileExistsError will be raised if it already - exists when opened for creating. Opening a file for creating implies - writing so this mode behaves in a similar way to 'w'. Add a '+' to the mode - to allow simultaneous reading and writing. A custom opener can be used by - passing a callable as *opener*. The underlying file descriptor for the file - object is then obtained by calling opener with (*name*, *flags*). - *opener* must return an open file descriptor (passing os.open as *opener* - results in functionality similar to passing None). + """Open a file. + + The mode can be 'r' (default), 'w', 'x' or 'a' for reading, + writing, exclusive creation or appending. The file will be created + if it doesn't exist when opened for writing or appending; it will be + truncated when opened for writing. A FileExistsError will be raised + if it already exists when opened for creating. Opening a file for + creating implies writing so this mode behaves in a similar way to + 'w'. Add a '+' to the mode to allow simultaneous reading and + writing. + + A custom opener can be used by passing a callable as *opener*. + The underlying file descriptor for the file object is then obtained + by calling opener with (*name*, *flags*). *opener* must return + an open file descriptor (passing os.open as *opener* results in + functionality similar to passing None). """ if self._fd >= 0: # Have to close the existing file first. + self._stat_atopen = None try: if self._closefd: os.close(self._fd) @@ -1573,18 +1588,15 @@ def __init__(self, file, mode='r', closefd=True, opener=None): os.set_inheritable(fd, False) self._closefd = closefd - fdfstat = os.fstat(fd) + self._stat_atopen = os.fstat(fd) try: - if stat.S_ISDIR(fdfstat.st_mode): + if stat.S_ISDIR(self._stat_atopen.st_mode): raise IsADirectoryError(errno.EISDIR, os.strerror(errno.EISDIR), file) except AttributeError: # Ignore the AttributeError if stat.S_ISDIR or errno.EISDIR # don't exist. pass - self._blksize = getattr(fdfstat, 'st_blksize', 0) - if self._blksize <= 1: - self._blksize = DEFAULT_BUFFER_SIZE if _setmode: # don't translate newlines (\r\n <=> \n) @@ -1601,6 +1613,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if e.errno != errno.ESPIPE: raise except: + self._stat_atopen = None if owned_fd is not None: os.close(owned_fd) raise @@ -1629,6 +1642,17 @@ def __repr__(self): return ('<%s name=%r mode=%r closefd=%r>' % (class_name, name, self.mode, self._closefd)) + @property + def _blksize(self): + if self._stat_atopen is None: + return DEFAULT_BUFFER_SIZE + + blksize = getattr(self._stat_atopen, "st_blksize", 0) + # WASI sets blsize to 0 + if not blksize: + return DEFAULT_BUFFER_SIZE + return blksize + def _checkReadable(self): if not self._readable: raise UnsupportedOperation('File not open for reading') @@ -1640,7 +1664,13 @@ def _checkWritable(self, msg=None): def read(self, size=None): """Read at most size bytes, returned as bytes. - Only makes one system call, so less data may be returned than requested + If size is less than 0, read all bytes in the file making + multiple read calls. See ``FileIO.readall``. + + Attempts to make only one system call, retrying only per + PEP 475 (EINTR). This means less data may be returned than + requested. + In non-blocking mode, returns None if no data is available. Return an empty bytes object at EOF. """ @@ -1656,52 +1686,64 @@ def read(self, size=None): def readall(self): """Read all data from the file, returned as bytes. - In non-blocking mode, returns as much as is immediately available, - or None if no data is available. Return an empty bytes object at EOF. + Reads until either there is an error or read() returns size 0 + (indicates EOF). If the file is already at EOF, returns an + empty bytes object. + + In non-blocking mode, returns as much data as could be read + before EAGAIN. If no data is available (EAGAIN is returned + before bytes are read) returns None. """ self._checkClosed() self._checkReadable() - bufsize = DEFAULT_BUFFER_SIZE - try: - pos = os.lseek(self._fd, 0, SEEK_CUR) - end = os.fstat(self._fd).st_size - if end >= pos: - bufsize = end - pos + 1 - except OSError: - pass + if self._stat_atopen is None or self._stat_atopen.st_size <= 0: + bufsize = DEFAULT_BUFFER_SIZE + else: + # In order to detect end of file, need a read() of at least 1 + # byte which returns size 0. Oversize the buffer by 1 byte so the + # I/O can be completed with two read() calls (one for all data, one + # for EOF) without needing to resize the buffer. + bufsize = self._stat_atopen.st_size + 1 - result = bytearray() - while True: - if len(result) >= bufsize: - bufsize = len(result) - bufsize += max(bufsize, DEFAULT_BUFFER_SIZE) - n = bufsize - len(result) - try: - chunk = os.read(self._fd, n) - except BlockingIOError: - if result: - break + if self._stat_atopen.st_size > 65536: + try: + pos = os.lseek(self._fd, 0, SEEK_CUR) + if self._stat_atopen.st_size >= pos: + bufsize = self._stat_atopen.st_size - pos + 1 + except OSError: + pass + + result = bytearray(bufsize) + bytes_read = 0 + try: + while n := os.readinto(self._fd, memoryview(result)[bytes_read:]): + bytes_read += n + if bytes_read >= len(result): + result.resize(_new_buffersize(bytes_read)) + except BlockingIOError: + if not bytes_read: return None - if not chunk: # reached the end of the file - break - result += chunk + assert len(result) - bytes_read >= 1, \ + "os.readinto buffer size 0 will result in erroneous EOF / returns 0" + result.resize(bytes_read) return bytes(result) - def readinto(self, b): + def readinto(self, buffer): """Same as RawIOBase.readinto().""" - m = memoryview(b).cast('B') - data = self.read(len(m)) - n = len(data) - m[:n] = data - return n + self._checkClosed() + self._checkReadable() + try: + return os.readinto(self._fd, buffer) + except BlockingIOError: + return None def write(self, b): """Write bytes b to file, return number written. Only makes one system call, so not all of the data may be written. - The number of bytes actually written is returned. In non-blocking mode, - returns None if the write would block. + The number of bytes actually written is returned. In non-blocking + mode, returns None if the write would block. """ self._checkClosed() self._checkWritable() @@ -1713,11 +1755,12 @@ def write(self, b): def seek(self, pos, whence=SEEK_SET): """Move to new file position. - Argument offset is a byte count. Optional argument whence defaults to - SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values - are SEEK_CUR or 1 (move relative to current position, positive or negative), - and SEEK_END or 2 (move relative to end of file, usually negative, although - many platforms allow seeking beyond the end of a file). + Argument offset is a byte count. Optional argument whence defaults + to SEEK_SET or 0 (offset from start of file, offset should be >= 0); + other values are SEEK_CUR or 1 (move relative to current position, + positive or negative), and SEEK_END or 2 (move relative to end of + file, usually negative, although many platforms allow seeking beyond + the end of a file). Note that not all file objects are seekable. """ @@ -1744,15 +1787,17 @@ def truncate(self, size=None): if size is None: size = self.tell() os.ftruncate(self._fd, size) + self._stat_atopen = None return size def close(self): """Close the file. - A closed file cannot be used for further I/O operations. close() may be - called more than once without error. + A closed file cannot be used for further I/O operations. + close() may be called more than once without error. """ if not self.closed: + self._stat_atopen = None try: if self._closefd and self._fd >= 0: os.close(self._fd) @@ -1791,6 +1836,21 @@ def isatty(self): self._checkClosed() return os.isatty(self._fd) + def _isatty_open_only(self): + """Checks whether the file is a TTY using an open-only optimization. + + TTYs are always character devices. If the interpreter knows a file is + not a character device when it would call ``isatty``, can skip that + call. Inside ``open()`` there is a fresh stat result that contains that + information. Use the stat result to skip a system call. Outside of that + context TOCTOU issues (the fd could be arbitrarily modified by + surrounding code). + """ + if (self._stat_atopen is not None + and not stat.S_ISCHR(self._stat_atopen.st_mode)): + return False + return os.isatty(self._fd) + @property def closefd(self): """True if the file descriptor will be closed by close().""" @@ -1829,8 +1889,8 @@ class TextIOBase(IOBase): def read(self, size=-1): """Read at most size characters from stream, where size is an int. - Read from underlying buffer until we have size characters or we hit EOF. - If size is negative or omitted, read until EOF. + Read from underlying buffer until we have size characters or we hit + EOF. If size is negative or omitted, read until EOF. Returns a string. """ @@ -1844,7 +1904,7 @@ def truncate(self, pos=None): """Truncate size to pos, where pos is an int.""" self._unsupported("truncate") - def readline(self): + def readline(self, size=-1, /): """Read until newline or EOF. Returns an empty string if EOF is hit immediately. @@ -2015,8 +2075,7 @@ def __init__(self, buffer, encoding=None, errors=None, newline=None, raise ValueError("invalid encoding: %r" % encoding) if not codecs.lookup(encoding)._is_text_encoding: - msg = ("%r is not a text encoding; " - "use codecs.open() to handle arbitrary codecs") + msg = "%r is not a text encoding" raise LookupError(msg % encoding) if errors is None: @@ -2524,9 +2583,12 @@ def read(self, size=None): size = size_index() decoder = self._decoder or self._get_decoder() if size < 0: + chunk = self.buffer.read() + if chunk is None: + raise BlockingIOError("Read returned None.") # Read everything. result = (self._get_decoded_chars() + - decoder.decode(self.buffer.read(), final=True)) + decoder.decode(chunk, final=True)) if self._snapshot is not None: self._set_decoded_chars('') self._snapshot = None diff --git a/Lib/_pylong.py b/Lib/_pylong.py index 4970eb3fa67..be1acd17ce3 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -45,10 +45,16 @@ # # and `mycache[lo]` replaces `base**lo` in the inner function. # -# While this does give minor speedups (a few percent at best), the primary -# intent is to simplify the functions using this, by eliminating the need for -# them to craft their own ad-hoc caching schemes. -def compute_powers(w, base, more_than, show=False): +# If an algorithm wants the powers of ceiling(w/2) instead of the floor, +# pass keyword argument `need_hi=True`. +# +# While this does give minor speedups (a few percent at best), the +# primary intent is to simplify the functions using this, by eliminating +# the need for them to craft their own ad-hoc caching schemes. +# +# See code near end of file for a block of code that can be enabled to +# run millions of tests. +def compute_powers(w, base, more_than, *, need_hi=False, show=False): seen = set() need = set() ws = {w} @@ -58,40 +64,70 @@ def compute_powers(w, base, more_than, show=False): continue seen.add(w) lo = w >> 1 - # only _need_ lo here; some other path may, or may not, need hi - need.add(lo) - ws.add(lo) - if w & 1: - ws.add(lo + 1) + hi = w - lo + # only _need_ one here; the other may, or may not, be needed + which = hi if need_hi else lo + need.add(which) + ws.add(which) + if lo != hi: + ws.add(w - which) + + # `need` is the set of exponents needed. To compute them all + # efficiently, possibly add other exponents to `extra`. The goal is + # to ensure that each exponent can be gotten from a smaller one via + # multiplying by the base, squaring it, or squaring and then + # multiplying by the base. + # + # If need_hi is False, this is already the case (w can always be + # gotten from w >> 1 via one of the squaring strategies). But we do + # the work anyway, just in case ;-) + # + # Note that speed is irrelevant. These loops are working on little + # ints (exponents) and go around O(log w) times. The total cost is + # insignificant compared to just one of the bigint multiplies. + cands = need.copy() + extra = set() + while cands: + w = max(cands) + cands.remove(w) + lo = w >> 1 + if lo > more_than and w-1 not in cands and lo not in cands: + extra.add(lo) + cands.add(lo) + assert need_hi or not extra d = {} - if not need: - return d - it = iter(sorted(need)) - first = next(it) - if show: - print("pow at", first) - d[first] = base ** first - for this in it: - if this - 1 in d: + for n in sorted(need | extra): + lo = n >> 1 + hi = n - lo + if n-1 in d: if show: - print("* base at", this) - d[this] = d[this - 1] * base # cheap - else: - lo = this >> 1 - hi = this - lo - assert lo in d + print("* base", end="") + result = d[n-1] * base # cheap! + elif lo in d: + # Multiplying a bigint by itself is about twice as fast + # in CPython provided it's the same object. if show: - print("square at", this) - # Multiplying a bigint by itself (same object!) is about twice - # as fast in CPython. - sq = d[lo] * d[lo] + print("square", end="") + result = d[lo] * d[lo] # same object if hi != lo: - assert hi == lo + 1 if show: - print(" and * base") - sq *= base - d[this] = sq + print(" * base", end="") + assert 2 * lo + 1 == n + result *= base + else: # rare + if show: + print("pow", end='') + result = base ** n + if show: + print(" at", n, "needed" if n in need else "extra") + d[n] = result + + assert need <= d.keys() + if excess := d.keys() - need: + assert need_hi + for n in excess: + del d[n] return d _unbounded_dec_context = decimal.getcontext().copy() @@ -211,6 +247,145 @@ def inner(a, b): return inner(0, len(s)) +# Asymptotically faster version, using the C decimal module. See +# comments at the end of the file. This uses decimal arithmetic to +# convert from base 10 to base 256. The latter is just a string of +# bytes, which CPython can convert very efficiently to a Python int. + +# log of 10 to base 256 with best-possible 53-bit precision. Obtained +# via: +# from mpmath import mp +# mp.prec = 1000 +# print(float(mp.log(10, 256)).hex()) +_LOG_10_BASE_256 = float.fromhex('0x1.a934f0979a371p-2') # about 0.415 + +# _spread is for internal testing. It maps a key to the number of times +# that condition obtained in _dec_str_to_int_inner: +# key 0 - quotient guess was right +# key 1 - quotient had to be boosted by 1, one time +# key 999 - one adjustment wasn't enough, so fell back to divmod +from collections import defaultdict +_spread = defaultdict(int) +del defaultdict + +def _dec_str_to_int_inner(s, *, GUARD=8): + # Yes, BYTELIM is "large". Large enough that CPython will usually + # use the Karatsuba _str_to_int_inner to convert the string. This + # allowed reducing the cutoff for calling _this_ function from 3.5M + # to 2M digits. We could almost certainly do even better by + # fine-tuning this and/or using a larger output base than 256. + BYTELIM = 100_000 + D = decimal.Decimal + result = bytearray() + # See notes at end of file for discussion of GUARD. + assert GUARD > 0 # if 0, `decimal` can blow up - .prec 0 not allowed + + def inner(n, w): + #assert n < D256 ** w # required, but too expensive to check + if w <= BYTELIM: + # XXX Stefan Pochmann discovered that, for 1024-bit ints, + # `int(Decimal)` took 2.5x longer than `int(str(Decimal))`. + # Worse, `int(Decimal) is still quadratic-time for much + # larger ints. So unless/until all that is repaired, the + # seemingly redundant `str(Decimal)` is crucial to speed. + result.extend(int(str(n)).to_bytes(w)) # big-endian default + return + w1 = w >> 1 + w2 = w - w1 + if 0: + # This is maximally clear, but "too slow". `decimal` + # division is asymptotically fast, but we have no way to + # tell it to reuse the high-precision reciprocal it computes + # for pow256[w2], so it has to recompute it over & over & + # over again :-( + hi, lo = divmod(n, pow256[w2][0]) + else: + p256, recip = pow256[w2] + # The integer part will have a number of digits about equal + # to the difference between the log10s of `n` and `pow256` + # (which, since these are integers, is roughly approximated + # by `.adjusted()`). That's the working precision we need, + ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD + hi = +n * +recip # unary `+` chops back to ctx.prec digits + ctx.prec = decimal.MAX_PREC + hi = hi.to_integral_value() # lose the fractional digits + lo = n - hi * p256 + # Because we've been uniformly rounding down, `hi` is a + # lower bound on the correct quotient. + assert lo >= 0 + # Adjust quotient up if needed. It usually isn't. In random + # testing on inputs through 5 billion digit strings, the + # test triggered once in about 200 thousand tries. + count = 0 + if lo >= p256: + count = 1 + lo -= p256 + hi += 1 + if lo >= p256: + # Complete correction via an exact computation. I + # believe it's not possible to get here provided + # GUARD >= 3. It's tested by reducing GUARD below + # that. + count = 999 + hi2, lo = divmod(lo, p256) + hi += hi2 + _spread[count] += 1 + # The assert should always succeed, but way too slow to keep + # enabled. + #assert hi, lo == divmod(n, pow256[w2][0]) + inner(hi, w1) + del hi # at top levels, can free a lot of RAM "early" + inner(lo, w2) + + # How many base 256 digits are needed?. Mathematically, exactly + # floor(log256(int(s))) + 1. There is no cheap way to compute this. + # But we can get an upper bound, and that's necessary for our error + # analysis to make sense. int(s) < 10**len(s), so the log needed is + # < log256(10**len(s)) = len(s) * log256(10). However, using + # finite-precision floating point for this, it's possible that the + # computed value is a little less than the true value. If the true + # value is at - or a little higher than - an integer, we can get an + # off-by-1 error too low. So we add 2 instead of 1 if chopping lost + # a fraction > 0.9. + + # The "WASI" test platform can complain about `len(s)` if it's too + # large to fit in its idea of "an index-sized integer". + lenS = s.__len__() + log_ub = lenS * _LOG_10_BASE_256 + log_ub_as_int = int(log_ub) + w = log_ub_as_int + 1 + (log_ub - log_ub_as_int > 0.9) + # And what if we've plain exhausted the limits of HW floats? We + # could compute the log to any desired precision using `decimal`, + # but it's not plausible that anyone will pass a string requiring + # trillions of bytes (unless they're just trying to "break things"). + if w.bit_length() >= 46: + # "Only" had < 53 - 46 = 7 bits to spare in IEEE-754 double. + raise ValueError(f"cannot convert string of len {lenS} to int") + with decimal.localcontext(_unbounded_dec_context) as ctx: + D256 = D(256) + pow256 = compute_powers(w, D256, BYTELIM, need_hi=True) + rpow256 = compute_powers(w, 1 / D256, BYTELIM, need_hi=True) + # We're going to do inexact, chopped arithmetic, multiplying by + # an approximation to the reciprocal of 256**i. We chop to get a + # lower bound on the true integer quotient. Our approximation is + # a lower bound, the multiplication is chopped too, and + # to_integral_value() is also chopped. + ctx.traps[decimal.Inexact] = 0 + ctx.rounding = decimal.ROUND_DOWN + for k, v in pow256.items(): + # No need to save much more precision in the reciprocal than + # the power of 256 has, plus some guard digits to absorb + # most relevant rounding errors. This is highly significant: + # 1/2**i has the same number of significant decimal digits + # as 5**i, generally over twice the number in 2**i, + ctx.prec = v.adjusted() + GUARD + 1 + # The unary "+" chops the reciprocal back to that precision. + pow256[k] = v, +rpow256[k] + del rpow256 # exact reciprocals no longer needed + ctx.prec = decimal.MAX_PREC + inner(D(s), w) + return int.from_bytes(result) + def int_from_string(s): """Asymptotically fast version of PyLong_FromString(), conversion of a string of decimal digits into an 'int'.""" @@ -219,7 +394,10 @@ def int_from_string(s): # and underscores, and stripped leading whitespace. The input can still # contain underscores and have trailing whitespace. s = s.rstrip().replace('_', '') - return _str_to_int_inner(s) + func = _str_to_int_inner + if len(s) >= 2_000_000 and _decimal is not None: + func = _dec_str_to_int_inner + return func(s) def str_to_int(s): """Asymptotically fast version of decimal string to 'int' conversion.""" @@ -352,7 +530,7 @@ def int_divmod(a, b): Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b). """ if b == 0: - raise ZeroDivisionError + raise ZeroDivisionError('division by zero') elif b < 0: q, r = int_divmod(-a, -b) return q, -r @@ -361,3 +539,191 @@ def int_divmod(a, b): return ~q, b + ~r else: return _divmod_pos(a, b) + + +# Notes on _dec_str_to_int_inner: +# +# Stefan Pochmann worked up a str->int function that used the decimal +# module to, in effect, convert from base 10 to base 256. This is +# "unnatural", in that it requires multiplying and dividing by large +# powers of 2, which `decimal` isn't naturally suited to. But +# `decimal`'s `*` and `/` are asymptotically superior to CPython's, so +# at _some_ point it could be expected to win. +# +# Alas, the crossover point was too high to be of much real interest. I +# (Tim) then worked on ways to replace its division with multiplication +# by a cached reciprocal approximation instead, fixing up errors +# afterwards. This reduced the crossover point significantly, +# +# I revisited the code, and found ways to improve and simplify it. The +# crossover point is at about 3.4 million digits now. +# +# About .adjusted() +# ----------------- +# Restrict to Decimal values x > 0. We don't use negative numbers in the +# code, and I don't want to have to keep typing, e.g., "absolute value". +# +# For convenience, I'll use `x.a` to mean `x.adjusted()`. x.a doesn't +# look at the digits of x, but instead returns an integer giving x's +# order of magnitude. These are equivalent: +# +# - x.a is the power-of-10 exponent of x's most significant digit. +# - x.a = the infinitely precise floor(log10(x)) +# - x can be written in this form, where f is a real with 1 <= f < 10: +# x = f * 10**x.a +# +# Observation; if x is an integer, len(str(x)) = x.a + 1. +# +# Lemma 1: (x * y).a = x.a + y.a, or one larger +# +# Proof: Write x = f * 10**x.a and y = g * 10**y.a, where f and g are in +# [1, 10). Then x*y = f*g * 10**(x.a + y.a), where 1 <= f*g < 100. If +# f*g < 10, (x*y).a is x.a+y.a. Else divide f*g by 10 to bring it back +# into [1, 10], and add 1 to the exponent to compensate. Then (x*y).a is +# x.a+y.a+1. +# +# Lemma 2: ceiling(log10(x/y)) <= x.a - y.a + 1 +# +# Proof: Express x and y as in Lemma 1. Then x/y = f/g * 10**(x.a - +# y.a), where 1/10 < f/g < 10. If 1 <= f/g, (x/y).a is x.a-y.a. Else +# multiply f/g by 10 to bring it back into [1, 10], and subtract 1 from +# the exponent to compensate. Then (x/y).a is x.a-y.a-1. So the largest +# (x/y).a can be is x.a-y.a. Since that's the floor of log10(x/y). the +# ceiling is at most 1 larger (with equality iff f/g = 1 exactly). +# +# GUARD digits +# ------------ +# We only want the integer part of divisions, so don't need to build +# the full multiplication tree. But using _just_ the number of +# digits expected in the integer part ignores too much. What's left +# out can have a very significant effect on the quotient. So we use +# GUARD additional digits. +# +# The default 8 is more than enough so no more than 1 correction step +# was ever needed for all inputs tried through 2.5 billion digits. In +# fact, I believe 3 guard digits are always enough - but the proof is +# very involved, so better safe than sorry. +# +# Short course: +# +# If prec is the decimal precision in effect, and we're rounding down, +# the result of an operation is exactly equal to the infinitely precise +# result times 1-e for some real e with 0 <= e < 10**(1-prec). In +# +# ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD +# hi = +n * +recip # unary `+` chops to ctx.prec digits +# +# we have 3 visible chopped operations, but there's also a 4th: +# precomputing a truncated `recip` as part of setup. +# +# So the computed product is exactly equal to the true product times +# (1-e1)*(1-e2)*(1-e3)*(1-e4); since the e's are all very small, an +# excellent approximation to the second factor is 1-(e1+e2+e3+e4) (the +# 2nd and higher order terms in the expanded product are too tiny to +# matter). If they're all as large as possible, that's +# +# 1 - 4*10**(1-prec). This, BTW, is all bog-standard FP error analysis. +# +# That implies the computed product is within 1 of the true product +# provided prec >= log10(true_product) + 1.602. +# +# Here are telegraphic details, rephrasing the initial condition in +# equivalent ways, step by step: +# +# prod - prod * (1 - 4*10**(1-prec)) <= 1 +# prod - prod + prod * 4*10**(1-prec)) <= 1 +# prod * 4*10**(1-prec)) <= 1 +# 10**(log10(prod)) * 4*10**(1-prec)) <= 1 +# 4*10**(1-prec+log10(prod))) <= 1 +# 10**(1-prec+log10(prod))) <= 1/4 +# 1-prec+log10(prod) <= log10(1/4) = -0.602 +# -prec <= -1.602 - log10(prod) +# prec >= log10(prod) + 1.602 +# +# The true product is the same as the true ratio n/p256. By Lemma 2 +# above, n.a - p256.a + 1 is an upper bound on the ceiling of +# log10(prod). Then 2 is the ceiling of 1.602. so n.a - p256.a + 3 is an +# upper bound on the right hand side of the inequality. Any prec >= that +# will work. +# +# But since this is just a sketch of a proof ;-), the code uses the +# empirically tested 8 instead of 3. 5 digits more or less makes no +# practical difference to speed - these ints are huge. And while +# increasing GUARD above 3 may not be necessary, every increase cuts the +# percentage of cases that need a correction at all. +# +# On Computing Reciprocals +# ------------------------ +# In general, the exact reciprocals we compute have over twice as many +# significant digits as needed. 1/256**i has the same number of +# significant decimal digits as 5**i. It's a significant waste of RAM +# to store all those unneeded digits. +# +# So we cut exact reciprocals back to the least precision that can +# be needed so that the error analysis above is valid, +# +# [Note: turns out it's very significantly faster to do it this way than +# to compute 1 / 256**i directly to the desired precision, because the +# power method doesn't require division. It's also faster than computing +# (1/256)**i directly to the desired precision - no material division +# there, but `compute_powers()` is much smarter about _how_ to compute +# all the powers needed than repeated applications of `**` - that +# function invokes `**` for at most the few smallest powers needed.] +# +# The hard part is that chopping back to a shorter width occurs +# _outside_ of `inner`. We can't know then what `prec` `inner()` will +# need. We have to pick, for each value of `w2`, the largest possible +# value `prec` can become when `inner()` is working on `w2`. +# +# This is the `prec` inner() uses: +# max(n.a - p256.a, 0) + GUARD +# and what setup uses (renaming its `v` to `p256` - same thing): +# p256.a + GUARD + 1 +# +# We need that the second is always at least as large as the first, +# which is the same as requiring +# +# n.a - 2 * p256.a <= 1 +# +# What's the largest n can be? n < 255**w = 256**(w2 + (w - w2)). The +# worst case in this context is when w ix even. and then w = 2*w2, so +# n < 256**(2*w2) = (256**w2)**2 = p256**2. By Lemma 1, then, n.a +# is at most p256.a + p256.a + 1. +# +# So the most n.a - 2 * p256.a can be is +# p256.a + p256.a + 1 - 2 * p256.a = 1. QED +# +# Note: an earlier version of the code split on floor(e/2) instead of on +# the ceiling. The worst case then is odd `w`, and a more involved proof +# was needed to show that adding 4 (instead of 1) may be necessary. +# Basically because, in that case, n may be up to 256 times larger than +# p256**2. Curiously enough, by splitting on the ceiling instead, +# nothing in any proof here actually depends on the output base (256). + +# Enable for brute-force testing of compute_powers(). This takes about a +# minute, because it tries millions of cases. +if 0: + def consumer(w, limit, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit: + return + if w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + exp = compute_powers(w, 1, limit, need_hi=need_hi) + assert exp.keys() == need + + from itertools import chain + for need_hi in (False, True): + for limit in (0, 1, 10, 100, 1_000, 10_000, 100_000): + for w in chain(range(1, 100_000), + (10**i for i in range(5, 30))): + consumer(w, limit, need_hi) diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py index 3fa992eee8e..9c66812e13a 100644 --- a/Lib/_pyrepl/__main__.py +++ b/Lib/_pyrepl/__main__.py @@ -1,6 +1,10 @@ # Important: don't add things to this module, as they will end up in the REPL's # default globals. Use _pyrepl.main instead. +# Avoid caching this file by linecache and incorrectly report tracebacks. +# See https://github.com/python/cpython/issues/129098. +__spec__ = __loader__ = None + if __name__ == "__main__": from .main import interactive_console as __pyrepl_interactive_console __pyrepl_interactive_console() diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py deleted file mode 100644 index d884f880f50..00000000000 --- a/Lib/_pyrepl/_minimal_curses.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Minimal '_curses' module, the low-level interface for curses module -which is not meant to be used directly. - -Based on ctypes. It's too incomplete to be really called '_curses', so -to use it, you have to import it and stick it in sys.modules['_curses'] -manually. - -Note that there is also a built-in module _minimal_curses which will -hide this one if compiled in. -""" - -import ctypes -import ctypes.util - - -class error(Exception): - pass - - -def _find_clib() -> str: - trylibs = ["ncursesw", "ncurses", "curses"] - - for lib in trylibs: - path = ctypes.util.find_library(lib) - if path: - return path - raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") - - -_clibpath = _find_clib() -clib = ctypes.cdll.LoadLibrary(_clibpath) - -clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] -clib.setupterm.restype = ctypes.c_int - -clib.tigetstr.argtypes = [ctypes.c_char_p] -clib.tigetstr.restype = ctypes.c_ssize_t - -clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] -clib.tparm.restype = ctypes.c_char_p - -OK = 0 -ERR = -1 - -# ____________________________________________________________ - - -def setupterm(termstr, fd): - err = ctypes.c_int(0) - result = clib.setupterm(termstr, fd, ctypes.byref(err)) - if result == ERR: - raise error("setupterm() failed (err=%d)" % err.value) - - -def tigetstr(cap): - if not isinstance(cap, bytes): - cap = cap.encode("ascii") - result = clib.tigetstr(cap) - if result == ERR: - return None - return ctypes.cast(result, ctypes.c_char_p).value - - -def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): - result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) - if result is None: - raise error("tparm() returned NULL") - return result diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py new file mode 100644 index 00000000000..2098d0a54ab --- /dev/null +++ b/Lib/_pyrepl/_module_completer.py @@ -0,0 +1,426 @@ +from __future__ import annotations + +import importlib +import os +import pkgutil +import sys +import token +import tokenize +from importlib.machinery import FileFinder +from io import StringIO +from contextlib import contextmanager +from dataclasses import dataclass +from itertools import chain +from tokenize import TokenInfo + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any, Iterable, Iterator, Mapping + + +HARDCODED_SUBMODULES = { + # Standard library submodules that are not detected by pkgutil.iter_modules + # but can be imported, so should be proposed in completion + "collections": ["abc"], + "os": ["path"], + "xml.parsers.expat": ["errors", "model"], +} + + +def make_default_module_completer() -> ModuleCompleter: + # Inside pyrepl, __package__ is set to None by default + return ModuleCompleter(namespace={'__package__': None}) + + +class ModuleCompleter: + """A completer for Python import statements. + + Examples: + - import + - import foo + - import foo. + - import foo as bar, baz + + - from + - from foo + - from foo import + - from foo import bar + - from foo import (bar as baz, qux + """ + + def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: + self.namespace = namespace or {} + self._global_cache: list[pkgutil.ModuleInfo] = [] + self._curr_sys_path: list[str] = sys.path[:] + self._stdlib_path = os.path.dirname(importlib.__path__[0]) + + def get_completions(self, line: str) -> list[str] | None: + """Return the next possible import completions for 'line'.""" + result = ImportParser(line).parse() + if not result: + return None + try: + return self.complete(*result) + except Exception: + # Some unexpected error occurred, make it look like + # no completions are available + return [] + + def complete(self, from_name: str | None, name: str | None) -> list[str]: + if from_name is None: + # import x.y.z + assert name is not None + path, prefix = self.get_path_and_prefix(name) + modules = self.find_modules(path, prefix) + return [self.format_completion(path, module) for module in modules] + + if name is None: + # from x.y.z + path, prefix = self.get_path_and_prefix(from_name) + modules = self.find_modules(path, prefix) + return [self.format_completion(path, module) for module in modules] + + # from x.y import z + return self.find_modules(from_name, name) + + def find_modules(self, path: str, prefix: str) -> list[str]: + """Find all modules under 'path' that start with 'prefix'.""" + modules = self._find_modules(path, prefix) + # Filter out invalid module names + # (for example those containing dashes that cannot be imported with 'import') + return [mod for mod in modules if mod.isidentifier()] + + def _find_modules(self, path: str, prefix: str) -> list[str]: + if not path: + # Top-level import (e.g. `import foo`` or `from foo`)` + builtin_modules = [name for name in sys.builtin_module_names + if self.is_suggestion_match(name, prefix)] + third_party_modules = [module.name for module in self.global_cache + if self.is_suggestion_match(module.name, prefix)] + return sorted(builtin_modules + third_party_modules) + + if path.startswith('.'): + # Convert relative path to absolute path + package = self.namespace.get('__package__', '') + path = self.resolve_relative_name(path, package) # type: ignore[assignment] + if path is None: + return [] + + modules: Iterable[pkgutil.ModuleInfo] = self.global_cache + imported_module = sys.modules.get(path.split('.')[0]) + if imported_module: + # Filter modules to those who name and specs match the + # imported module to avoid invalid suggestions + spec = imported_module.__spec__ + if spec: + modules = [mod for mod in modules + if mod.name == spec.name + and mod.module_finder.find_spec(mod.name, None) == spec] + else: + modules = [] + + is_stdlib_import: bool | None = None + for segment in path.split('.'): + modules = [mod_info for mod_info in modules + if mod_info.ispkg and mod_info.name == segment] + if is_stdlib_import is None: + # Top-level import decide if we import from stdlib or not + is_stdlib_import = all( + self._is_stdlib_module(mod_info) for mod_info in modules + ) + modules = self.iter_submodules(modules) + + module_names = [module.name for module in modules] + if is_stdlib_import: + module_names.extend(HARDCODED_SUBMODULES.get(path, ())) + return [module_name for module_name in module_names + if self.is_suggestion_match(module_name, prefix)] + + def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: + return (isinstance(module_info.module_finder, FileFinder) + and module_info.module_finder.path == self._stdlib_path) + + def is_suggestion_match(self, module_name: str, prefix: str) -> bool: + if prefix: + return module_name.startswith(prefix) + # For consistency with attribute completion, which + # does not suggest private attributes unless requested. + return not module_name.startswith("_") + + def iter_submodules(self, parent_modules: list[pkgutil.ModuleInfo]) -> Iterator[pkgutil.ModuleInfo]: + """Iterate over all submodules of the given parent modules.""" + specs = [info.module_finder.find_spec(info.name, None) + for info in parent_modules if info.ispkg] + search_locations = set(chain.from_iterable( + getattr(spec, 'submodule_search_locations', []) + for spec in specs if spec + )) + return pkgutil.iter_modules(search_locations) + + def get_path_and_prefix(self, dotted_name: str) -> tuple[str, str]: + """ + Split a dotted name into an import path and a + final prefix that is to be completed. + + Examples: + 'foo.bar' -> 'foo', 'bar' + 'foo.' -> 'foo', '' + '.foo' -> '.', 'foo' + """ + if '.' not in dotted_name: + return '', dotted_name + if dotted_name.startswith('.'): + stripped = dotted_name.lstrip('.') + dots = '.' * (len(dotted_name) - len(stripped)) + if '.' not in stripped: + return dots, stripped + path, prefix = stripped.rsplit('.', 1) + return dots + path, prefix + path, prefix = dotted_name.rsplit('.', 1) + return path, prefix + + def format_completion(self, path: str, module: str) -> str: + if path == '' or path.endswith('.'): + return f'{path}{module}' + return f'{path}.{module}' + + def resolve_relative_name(self, name: str, package: str) -> str | None: + """Resolve a relative module name to an absolute name. + + Example: resolve_relative_name('.foo', 'bar') -> 'bar.foo' + """ + # taken from importlib._bootstrap + level = 0 + for character in name: + if character != '.': + break + level += 1 + bits = package.rsplit('.', level - 1) + if len(bits) < level: + return None + base = bits[0] + name = name[level:] + return f'{base}.{name}' if name else base + + @property + def global_cache(self) -> list[pkgutil.ModuleInfo]: + """Global module cache""" + if not self._global_cache or self._curr_sys_path != sys.path: + self._curr_sys_path = sys.path[:] + self._global_cache = list(pkgutil.iter_modules()) + return self._global_cache + + +class ImportParser: + """ + Parses incomplete import statements that are + suitable for autocomplete suggestions. + + Examples: + - import foo -> Result(from_name=None, name='foo') + - import foo. -> Result(from_name=None, name='foo.') + - from foo -> Result(from_name='foo', name=None) + - from foo import bar -> Result(from_name='foo', name='bar') + - from .foo import ( -> Result(from_name='.foo', name='') + + Note that the parser works in reverse order, starting from the + last token in the input string. This makes the parser more robust + when parsing multiple statements. + """ + _ignored_tokens = { + token.INDENT, token.DEDENT, token.COMMENT, + token.NL, token.NEWLINE, token.ENDMARKER + } + _keywords = {'import', 'from', 'as'} + + def __init__(self, code: str) -> None: + self.code = code + tokens = [] + try: + for t in tokenize.generate_tokens(StringIO(code).readline): + if t.type not in self._ignored_tokens: + tokens.append(t) + except tokenize.TokenError as e: + if 'unexpected EOF' not in str(e): + # unexpected EOF is fine, since we're parsing an + # incomplete statement, but other errors are not + # because we may not have all the tokens so it's + # safer to bail out + tokens = [] + except SyntaxError: + tokens = [] + self.tokens = TokenQueue(tokens[::-1]) + + def parse(self) -> tuple[str | None, str | None] | None: + if not (res := self._parse()): + return None + return res.from_name, res.name + + def _parse(self) -> Result | None: + with self.tokens.save_state(): + return self.parse_from_import() + with self.tokens.save_state(): + return self.parse_import() + + def parse_import(self) -> Result: + if self.code.rstrip().endswith('import') and self.code.endswith(' '): + return Result(name='') + if self.tokens.peek_string(','): + name = '' + else: + if self.code.endswith(' '): + raise ParseError('parse_import') + name = self.parse_dotted_name() + if name.startswith('.'): + raise ParseError('parse_import') + while self.tokens.peek_string(','): + self.tokens.pop() + self.parse_dotted_as_name() + if self.tokens.peek_string('import'): + return Result(name=name) + raise ParseError('parse_import') + + def parse_from_import(self) -> Result: + stripped = self.code.rstrip() + if stripped.endswith('import') and self.code.endswith(' '): + return Result(from_name=self.parse_empty_from_import(), name='') + if stripped.endswith('from') and self.code.endswith(' '): + return Result(from_name='') + if self.tokens.peek_string('(') or self.tokens.peek_string(','): + return Result(from_name=self.parse_empty_from_import(), name='') + if self.code.endswith(' '): + raise ParseError('parse_from_import') + name = self.parse_dotted_name() + if '.' in name: + self.tokens.pop_string('from') + return Result(from_name=name) + if self.tokens.peek_string('from'): + return Result(from_name=name) + from_name = self.parse_empty_from_import() + return Result(from_name=from_name, name=name) + + def parse_empty_from_import(self) -> str: + if self.tokens.peek_string(','): + self.tokens.pop() + self.parse_as_names() + if self.tokens.peek_string('('): + self.tokens.pop() + self.tokens.pop_string('import') + return self.parse_from() + + def parse_from(self) -> str: + from_name = self.parse_dotted_name() + self.tokens.pop_string('from') + return from_name + + def parse_dotted_as_name(self) -> str: + self.tokens.pop_name() + if self.tokens.peek_string('as'): + self.tokens.pop() + with self.tokens.save_state(): + return self.parse_dotted_name() + + def parse_dotted_name(self) -> str: + name = [] + if self.tokens.peek_string('.'): + name.append('.') + self.tokens.pop() + if (self.tokens.peek_name() + and (tok := self.tokens.peek()) + and tok.string not in self._keywords): + name.append(self.tokens.pop_name()) + if not name: + raise ParseError('parse_dotted_name') + while self.tokens.peek_string('.'): + name.append('.') + self.tokens.pop() + if (self.tokens.peek_name() + and (tok := self.tokens.peek()) + and tok.string not in self._keywords): + name.append(self.tokens.pop_name()) + else: + break + + while self.tokens.peek_string('.'): + name.append('.') + self.tokens.pop() + return ''.join(name[::-1]) + + def parse_as_names(self) -> None: + self.parse_as_name() + while self.tokens.peek_string(','): + self.tokens.pop() + self.parse_as_name() + + def parse_as_name(self) -> None: + self.tokens.pop_name() + if self.tokens.peek_string('as'): + self.tokens.pop() + self.tokens.pop_name() + + +class ParseError(Exception): + pass + + +@dataclass(frozen=True) +class Result: + from_name: str | None = None + name: str | None = None + + +class TokenQueue: + """Provides helper functions for working with a sequence of tokens.""" + + def __init__(self, tokens: list[TokenInfo]) -> None: + self.tokens: list[TokenInfo] = tokens + self.index: int = 0 + self.stack: list[int] = [] + + @contextmanager + def save_state(self) -> Any: + try: + self.stack.append(self.index) + yield + except ParseError: + self.index = self.stack.pop() + else: + self.stack.pop() + + def __bool__(self) -> bool: + return self.index < len(self.tokens) + + def peek(self) -> TokenInfo | None: + if not self: + return None + return self.tokens[self.index] + + def peek_name(self) -> bool: + if not (tok := self.peek()): + return False + return tok.type == token.NAME + + def pop_name(self) -> str: + tok = self.pop() + if tok.type != token.NAME: + raise ParseError('pop_name') + return tok.string + + def peek_string(self, string: str) -> bool: + if not (tok := self.peek()): + return False + return tok.string == string + + def pop_string(self, string: str) -> str: + tok = self.pop() + if tok.string != string: + raise ParseError('pop_string') + return tok.string + + def pop(self) -> TokenInfo: + if not self: + raise ParseError('pop') + tok = self.tokens[self.index] + self.index += 1 + return tok diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py new file mode 100644 index 00000000000..0589a0f437e --- /dev/null +++ b/Lib/_pyrepl/base_eventqueue.py @@ -0,0 +1,110 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +OS-independent base for an event and VT sequence scanner + +See unix_eventqueue and windows_eventqueue for subclasses. +""" + +from collections import deque + +from . import keymap +from .console import Event +from .trace import trace + +class BaseEventQueue: + def __init__(self, encoding: str, keymap_dict: dict[bytes, str]) -> None: + self.compiled_keymap = keymap.compile_keymap(keymap_dict) + self.keymap = self.compiled_keymap + trace("keymap {k!r}", k=self.keymap) + self.encoding = encoding + self.events: deque[Event] = deque() + self.buf = bytearray() + + def get(self) -> Event | None: + """ + Retrieves the next event from the queue. + """ + if self.events: + return self.events.popleft() + else: + return None + + def empty(self) -> bool: + """ + Checks if the queue is empty. + """ + return not self.events + + def flush_buf(self) -> bytearray: + """ + Flushes the buffer and returns its contents. + """ + old = self.buf + self.buf = bytearray() + return old + + def insert(self, event: Event) -> None: + """ + Inserts an event into the queue. + """ + trace('added event {event}', event=event) + self.events.append(event) + + def push(self, char: int | bytes) -> None: + """ + Processes a character by updating the buffer and handling special key mappings. + """ + assert isinstance(char, (int, bytes)) + ord_char = char if isinstance(char, int) else ord(char) + char = ord_char.to_bytes() + self.buf.append(ord_char) + + if char in self.keymap: + if self.keymap is self.compiled_keymap: + # sanity check, buffer is empty when a special key comes + assert len(self.buf) == 1 + k = self.keymap[char] + trace('found map {k!r}', k=k) + if isinstance(k, dict): + self.keymap = k + else: + self.insert(Event('key', k, bytes(self.flush_buf()))) + self.keymap = self.compiled_keymap + + elif self.buf and self.buf[0] == 27: # escape + # escape sequence not recognized by our keymap: propagate it + # outside so that i can be recognized as an M-... key (see also + # the docstring in keymap.py + trace('unrecognized escape sequence, propagating...') + self.keymap = self.compiled_keymap + self.insert(Event('key', '\033', b'\033')) + for _c in self.flush_buf()[1:]: + self.push(_c) + + else: + try: + decoded = bytes(self.buf).decode(self.encoding) + except UnicodeError: + return + else: + self.insert(Event('key', decoded, bytes(self.flush_buf()))) + self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 503ca1da329..10127e58897 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -21,6 +21,7 @@ from __future__ import annotations import os +import time # Categories of actions: # killing @@ -31,6 +32,7 @@ # finishing # [completion] +from .trace import trace # types if False: @@ -368,6 +370,13 @@ def do(self) -> None: r = self.reader text = self.event * r.get_arg() r.insert(text) + if r.paste_mode: + data = "" + ev = r.console.getpending() + data += ev.data + if data: + r.insert(data) + r.last_refresh_cache.invalidated = True class insert_nl(EditCommand): @@ -411,14 +420,17 @@ class delete(EditCommand): def do(self) -> None: r = self.reader b = r.buffer - if ( - r.pos == 0 - and len(b) == 0 # this is something of a hack - and self.event[-1] == "\004" - ): - r.update_screen() - r.console.finish() - raise EOFError + if self.event[-1] == "\004": + if b and b[-1].endswith("\n"): + self.finish = True + elif ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + ): + r.update_screen() + r.console.finish() + raise EOFError + for i in range(r.get_arg()): if r.pos != len(b): del b[r.pos] @@ -437,7 +449,7 @@ def do(self) -> None: import _sitebuiltins with self.reader.suspend(): - self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment, call-arg] + self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment] class invalid_key(Command): @@ -456,7 +468,7 @@ def do(self) -> None: class show_history(Command): def do(self) -> None: from .pager import get_pager - from site import gethistoryfile # type: ignore[attr-defined] + from site import gethistoryfile history = os.linesep.join(self.reader.history[:]) self.reader.console.restore() @@ -471,19 +483,23 @@ def do(self) -> None: class paste_mode(Command): - def do(self) -> None: self.reader.paste_mode = not self.reader.paste_mode self.reader.dirty = True -class enable_bracketed_paste(Command): - def do(self) -> None: - self.reader.paste_mode = True - self.reader.in_bracketed_paste = True - -class disable_bracketed_paste(Command): - def do(self) -> None: - self.reader.paste_mode = False - self.reader.in_bracketed_paste = False - self.reader.dirty = True +class perform_bracketed_paste(Command): + def do(self) -> None: + done = "\x1b[201~" + data = "" + start = time.time() + while done not in data: + ev = self.reader.console.getpending() + data += ev.data + trace( + "bracketed pasting of {l} chars done in {s:.2f}s", + l=len(data), + s=time.time() - start, + ) + self.reader.insert(data.replace(done, "")) + self.reader.last_refresh_cache.invalidated = True diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index 9a005281dab..9d2d43be514 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -91,7 +91,7 @@ def build_menu( # D E F B E # G C F # - # "fill" the table with empty words, so we always have the same amout + # "fill" the table with empty words, so we always have the same amount # of rows for each column missing = cols*rows - len(wordlist) wordlist = wordlist + ['']*missing @@ -293,3 +293,7 @@ def get_stem(self) -> str: def get_completions(self, stem: str) -> list[str]: return [] + + def get_line(self) -> str: + """Return the current line until the cursor position.""" + return ''.join(self.buffer[:self.pos]) diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index 0d78890b4f4..8956fb1242e 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -19,11 +19,12 @@ from __future__ import annotations -import _colorize # type: ignore[import-not-found] +import _colorize from abc import ABC, abstractmethod import ast import code +import linecache from dataclasses import dataclass, field import os.path import sys @@ -152,6 +153,8 @@ def repaint(self) -> None: ... class InteractiveColoredConsole(code.InteractiveConsole): + STATEMENT_FAILED = object() + def __init__( self, locals: dict[str, object] | None = None, @@ -159,7 +162,7 @@ def __init__( *, local_exit: bool = False, ) -> None: - super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] + super().__init__(locals=locals, filename=filename, local_exit=local_exit) self.can_colorize = _colorize.can_colorize() def showsyntaxerror(self, filename=None, **kwargs): @@ -173,6 +176,16 @@ def _excepthook(self, typ, value, tb): limit=traceback.BUILTIN_EXCEPTION_LIMIT) self.write(''.join(lines)) + def runcode(self, code): + try: + exec(code, self.locals) + except SystemExit: + raise + except BaseException: + self.showtraceback() + return self.STATEMENT_FAILED + return None + def runsource(self, source, filename="", symbol="single"): try: tree = self.compile.compiler( @@ -193,6 +206,7 @@ def runsource(self, source, filename="", symbol="single"): item = wrapper([stmt]) try: code = self.compile.compiler(item, filename, the_symbol) + linecache._register_code(code, source, filename) except SyntaxError as e: if e.args[0] == "'await' outside function": python = os.path.basename(sys.executable) @@ -209,5 +223,7 @@ def runsource(self, source, filename="", symbol="single"): if code is None: return True - self.runcode(code) + result = self.runcode(code) + if result is self.STATEMENT_FAILED: + break return False diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py deleted file mode 100644 index 3a624d9f683..00000000000 --- a/Lib/_pyrepl/curses.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2000-2010 Michael Hudson-Doyle -# Armin Rigo -# -# All Rights Reserved -# -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose is hereby granted without fee, -# provided that the above copyright notice appear in all copies and -# that both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - -try: - import _curses -except ImportError: - try: - import curses as _curses # type: ignore[no-redef] - except ImportError: - from . import _minimal_curses as _curses # type: ignore[no-redef] - -setupterm = _curses.setupterm -tigetstr = _curses.tigetstr -tparm = _curses.tparm -error = _curses.error diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py index 0468b9a2670..8d5bd183f21 100644 --- a/Lib/_pyrepl/fancy_termios.py +++ b/Lib/_pyrepl/fancy_termios.py @@ -20,19 +20,25 @@ import termios +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import cast +else: + cast = lambda typ, val: val + + class TermState: - def __init__(self, tuples): - ( - self.iflag, - self.oflag, - self.cflag, - self.lflag, - self.ispeed, - self.ospeed, - self.cc, - ) = tuples + def __init__(self, attrs: list[int | list[bytes]]) -> None: + self.iflag = cast(int, attrs[0]) + self.oflag = cast(int, attrs[1]) + self.cflag = cast(int, attrs[2]) + self.lflag = cast(int, attrs[3]) + self.ispeed = cast(int, attrs[4]) + self.ospeed = cast(int, attrs[5]) + self.cc = cast(list[bytes], attrs[6]) - def as_list(self): + def as_list(self) -> list[int | list[bytes]]: return [ self.iflag, self.oflag, @@ -45,32 +51,32 @@ def as_list(self): self.cc[:], ] - def copy(self): + def copy(self) -> "TermState": return self.__class__(self.as_list()) -def tcgetattr(fd): +def tcgetattr(fd: int) -> TermState: return TermState(termios.tcgetattr(fd)) -def tcsetattr(fd, when, attrs): +def tcsetattr(fd: int, when: int, attrs: TermState) -> None: termios.tcsetattr(fd, when, attrs.as_list()) class Term(TermState): TS__init__ = TermState.__init__ - def __init__(self, fd=0): + def __init__(self, fd: int = 0) -> None: self.TS__init__(termios.tcgetattr(fd)) self.fd = fd - self.stack = [] + self.stack: list[list[int | list[bytes]]] = [] - def save(self): + def save(self) -> None: self.stack.append(self.as_list()) - def set(self, when=termios.TCSANOW): + def set(self, when: int = termios.TCSANOW) -> None: termios.tcsetattr(self.fd, when, self.as_list()) - def restore(self): + def restore(self) -> None: self.TS__init__(self.stack.pop()) self.set() diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py index 2fb03d19523..d11df4b5164 100644 --- a/Lib/_pyrepl/keymap.py +++ b/Lib/_pyrepl/keymap.py @@ -30,20 +30,20 @@ pyrepl uses its own keyspec format that is meant to be a strict superset of readline's KEYSEQ format. This means that if a spec is found that readline accepts that this doesn't, it should be logged as a bug. Note that this means -we're using the `\\C-o' style of readline's keyspec, not the `Control-o' sort. +we're using the '\\C-o' style of readline's keyspec, not the 'Control-o' sort. The extension to readline is that the sequence \\ denotes the sequence of characters produced by hitting KEY. Examples: -`a' - what you get when you hit the `a' key -`\\EOA' - Escape - O - A (up, on my terminal) -`\\' - the up arrow key -`\\' - ditto (keynames are case-insensitive) -`\\C-o', `\\c-o' - control-o -`\\M-.' - meta-period -`\\E.' - ditto (that's how meta works for pyrepl) -`\\', `\\', `\\t', `\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' +'a' - what you get when you hit the 'a' key +'\\EOA' - Escape - O - A (up, on my terminal) +'\\' - the up arrow key +'\\' - ditto (keynames are case-insensitive) +'\\C-o', '\\c-o' - control-o +'\\M-.' - meta-period +'\\E.' - ditto (that's how meta works for pyrepl) +'\\', '\\', '\\t', '\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' - all of these are the tab character. """ diff --git a/Lib/_pyrepl/main.py b/Lib/_pyrepl/main.py index a6f824dcc4a..447eb1e551e 100644 --- a/Lib/_pyrepl/main.py +++ b/Lib/_pyrepl/main.py @@ -1,6 +1,7 @@ import errno import os import sys +import types CAN_USE_PYREPL: bool @@ -29,12 +30,10 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): print(FAIL_REASON, file=sys.stderr) return sys._baserepl() - if mainmodule: - namespace = mainmodule.__dict__ - else: - import __main__ - namespace = __main__.__dict__ - namespace.pop("__pyrepl_interactive_console", None) + if not mainmodule: + mainmodule = types.ModuleType("__main__") + + namespace = mainmodule.__dict__ # sys._baserepl() above does this internally, we do it here startup_path = os.getenv("PYTHONSTARTUP") diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini index 395f5945ab7..9375a55b53c 100644 --- a/Lib/_pyrepl/mypy.ini +++ b/Lib/_pyrepl/mypy.ini @@ -4,8 +4,9 @@ [mypy] files = Lib/_pyrepl +mypy_path = $MYPY_CONFIG_FILE_DIR/../../Misc/mypy explicit_package_bases = True -python_version = 3.12 +python_version = 3.13 platform = linux pretty = True diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index dc26bfd3a34..f0116e742d2 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -22,15 +22,13 @@ from __future__ import annotations import sys +import _colorize from contextlib import contextmanager from dataclasses import dataclass, field, fields -import unicodedata -from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] - from . import commands, console, input -from .utils import ANSI_ESCAPE_SEQUENCE, wlen, str_width +from .utils import wlen, unbracket, disp_str, gen_colors, THEME from .trace import trace @@ -39,38 +37,7 @@ from .types import Callback, SimpleContextManager, KeySpec, CommandName -def disp_str(buffer: str) -> tuple[str, list[int]]: - """disp_str(buffer:string) -> (string, [int]) - - Return the string that should be the printed representation of - |buffer| and a list detailing where the characters of |buffer| - get used up. E.g.: - - >>> disp_str(chr(3)) - ('^C', [1, 0]) - - """ - b: list[int] = [] - s: list[str] = [] - for c in buffer: - if c == '\x1a': - s.append(c) - b.append(2) - elif ord(c) < 128: - s.append(c) - b.append(1) - elif unicodedata.category(c).startswith("C"): - c = r"\u%04x" % ord(c) - s.append(c) - b.extend([0] * (len(c) - 1)) - else: - s.append(c) - b.append(str_width(c)) - return "".join(s), b - - -# syntax classes: - +# syntax classes SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3) @@ -136,8 +103,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]: (r"\M-9", "digit-arg"), (r"\M-\n", "accept"), ("\\\\", "self-insert"), - (r"\x1b[200~", "enable_bracketed_paste"), - (r"\x1b[201~", "disable_bracketed_paste"), + (r"\x1b[200~", "perform-bracketed-paste"), (r"\x03", "ctrl-c"), ] + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] @@ -175,20 +141,21 @@ class Reader: Instance variables of note include: * buffer: - A *list* (*not* a string at the moment :-) containing all the - characters that have been entered. + A per-character list containing all the characters that have been + entered. Does not include color information. * console: Hopefully encapsulates the OS dependent stuff. * pos: - A 0-based index into `buffer' for where the insertion point + A 0-based index into 'buffer' for where the insertion point is. * screeninfo: - Ahem. This list contains some info needed to move the - insertion point around reasonably efficiently. + A list of screen position tuples. Each list element is a tuple + representing information on visible line length for a given line. + Allows for efficient skipping of color escape sequences. * cxy, lxy: the position of the insertion point in screen ... * syntax_table: - Dictionary mapping characters to `syntax class'; read the + Dictionary mapping characters to 'syntax class'; read the emacs docs to see what this means :-) * commands: Dictionary mapping command names to command classes. @@ -234,7 +201,6 @@ class Reader: dirty: bool = False finished: bool = False paste_mode: bool = False - in_bracketed_paste: bool = False commands: dict[str, type[Command]] = field(default_factory=make_default_commands) last_command: type[Command] | None = None syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) @@ -252,7 +218,6 @@ class Reader: ## cached metadata to speed up screen refreshes @dataclass class RefreshCache: - in_bracketed_paste: bool = False screen: list[str] = field(default_factory=list) screeninfo: list[tuple[int, list[int]]] = field(init=False) line_end_offsets: list[int] = field(default_factory=list) @@ -266,7 +231,6 @@ def update_cache(self, screen: list[str], screeninfo: list[tuple[int, list[int]]], ) -> None: - self.in_bracketed_paste = reader.in_bracketed_paste self.screen = screen.copy() self.screeninfo = screeninfo.copy() self.pos = reader.pos @@ -279,8 +243,7 @@ def valid(self, reader: Reader) -> bool: return False dimensions = reader.console.width, reader.console.height dimensions_changed = dimensions != self.dimensions - paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste - return not (dimensions_changed or paste_changed) + return not dimensions_changed def get_cached_location(self, reader: Reader) -> tuple[int, int]: if self.invalidated: @@ -310,7 +273,7 @@ def __post_init__(self) -> None: self.screeninfo = [(0, [])] self.cxy = self.pos2xy() self.lxy = (self.pos, 0) - self.can_colorize = can_colorize() + self.can_colorize = _colorize.can_colorize() self.last_refresh_cache.screeninfo = self.screeninfo self.last_refresh_cache.pos = self.pos @@ -348,13 +311,17 @@ def calc_screen(self) -> list[str]: prompt_from_cache = (offset and self.buffer[offset - 1] != "\n") + if self.can_colorize: + colors = list(gen_colors(self.get_unicode())) + else: + colors = None + trace("colors = {colors}", colors=colors) lines = "".join(self.buffer[offset:]).split("\n") - cursor_found = False lines_beyond_cursor = 0 for ln, line in enumerate(lines, num_common_lines): - ll = len(line) - if 0 <= pos <= ll: + line_len = len(line) + if 0 <= pos <= line_len: self.lxy = pos, ln cursor_found = True elif cursor_found: @@ -368,34 +335,33 @@ def calc_screen(self) -> list[str]: prompt_from_cache = False prompt = "" else: - prompt = self.get_prompt(ln, ll >= pos >= 0) + prompt = self.get_prompt(ln, line_len >= pos >= 0) while "\n" in prompt: pre_prompt, _, prompt = prompt.partition("\n") last_refresh_line_end_offsets.append(offset) screen.append(pre_prompt) screeninfo.append((0, [])) - pos -= ll + 1 - prompt, lp = self.process_prompt(prompt) - l, l2 = disp_str(line) - wrapcount = (wlen(l) + lp) // self.console.width - if wrapcount == 0: - offset += ll + 1 # Takes all of the line plus the newline + pos -= line_len + 1 + prompt, prompt_len = self.process_prompt(prompt) + chars, char_widths = disp_str(line, colors, offset) + wrapcount = (sum(char_widths) + prompt_len) // self.console.width + if wrapcount == 0 or not char_widths: + offset += line_len + 1 # Takes all of the line plus the newline last_refresh_line_end_offsets.append(offset) - screen.append(prompt + l) - screeninfo.append((lp, l2)) + screen.append(prompt + "".join(chars)) + screeninfo.append((prompt_len, char_widths)) else: - i = 0 - while l: - prelen = lp if i == 0 else 0 + pre = prompt + prelen = prompt_len + for wrap in range(wrapcount + 1): index_to_wrap_before = 0 column = 0 - for character_width in l2: - if column + character_width >= self.console.width - prelen: + for char_width in char_widths: + if column + char_width + prelen >= self.console.width: break index_to_wrap_before += 1 - column += character_width - pre = prompt if i == 0 else "" - if len(l) > index_to_wrap_before: + column += char_width + if len(chars) > index_to_wrap_before: offset += index_to_wrap_before post = "\\" after = [1] @@ -404,11 +370,14 @@ def calc_screen(self) -> list[str]: post = "" after = [] last_refresh_line_end_offsets.append(offset) - screen.append(pre + l[:index_to_wrap_before] + post) - screeninfo.append((prelen, l2[:index_to_wrap_before] + after)) - l = l[index_to_wrap_before:] - l2 = l2[index_to_wrap_before:] - i += 1 + render = pre + "".join(chars[:index_to_wrap_before]) + post + render_widths = char_widths[:index_to_wrap_before] + after + screen.append(render) + screeninfo.append((prelen, render_widths)) + chars = chars[index_to_wrap_before:] + char_widths = char_widths[index_to_wrap_before:] + pre = "" + prelen = 0 self.screeninfo = screeninfo self.cxy = self.pos2xy() if self.msg: @@ -421,42 +390,15 @@ def calc_screen(self) -> list[str]: @staticmethod def process_prompt(prompt: str) -> tuple[str, int]: - """Process the prompt. - - This means calculate the length of the prompt. The character \x01 - and \x02 are used to bracket ANSI control sequences and need to be - excluded from the length calculation. So also a copy of the prompt - is returned with these control characters removed.""" + r"""Return a tuple with the prompt string and its visible length. - # The logic below also ignores the length of common escape - # sequences if they were not explicitly within \x01...\x02. - # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) - - # wlen from utils already excludes ANSI_ESCAPE_SEQUENCE chars, - # which breaks the logic below so we redefine it here. - def wlen(s: str) -> int: - return sum(str_width(i) for i in s) - - out_prompt = "" - l = wlen(prompt) - pos = 0 - while True: - s = prompt.find("\x01", pos) - if s == -1: - break - e = prompt.find("\x02", s) - if e == -1: - break - # Found start and end brackets, subtract from string length - l = l - (e - s + 1) - keep = prompt[pos:s] - l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) - out_prompt += keep + prompt[s + 1 : e] - pos = e + 1 - keep = prompt[pos:] - l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) - out_prompt += keep - return out_prompt, l + The prompt string has the zero-width brackets recognized by shells + (\x01 and \x02) removed. The length ignores anything between those + brackets as well as any ANSI escape sequences. + """ + out_prompt = unbracket(prompt, including_content=False) + visible_prompt = unbracket(prompt, including_content=True) + return out_prompt, wlen(visible_prompt) def bow(self, p: int | None = None) -> int: """Return the 0-based index of the word break preceding p most @@ -525,7 +467,7 @@ def max_row(self) -> int: def get_arg(self, default: int = 1) -> int: """Return any prefix argument that the user has supplied, - returning `default' if there is None. Defaults to 1. + returning 'default' if there is None. Defaults to 1. """ if self.arg is None: return default @@ -533,10 +475,10 @@ def get_arg(self, default: int = 1) -> int: def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: """Return what should be in the left-hand margin for line - `lineno'.""" + 'lineno'.""" if self.arg is not None and cursor_on_line: prompt = f"(arg: {self.arg}) " - elif self.paste_mode and not self.in_bracketed_paste: + elif self.paste_mode: prompt = "(paste) " elif "\n" in self.buffer: if lineno == 0: @@ -549,7 +491,8 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: prompt = self.ps1 if self.can_colorize: - prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}" + t = THEME() + prompt = f"{t.prompt}{prompt}{t.reset}" return prompt def push_input_trans(self, itrans: input.KeymapTranslator) -> None: @@ -564,9 +507,9 @@ def setpos_from_xy(self, x: int, y: int) -> None: pos = 0 i = 0 while i < y: - prompt_len, character_widths = self.screeninfo[i] - offset = len(character_widths) - character_widths.count(0) - in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width + prompt_len, char_widths = self.screeninfo[i] + offset = len(char_widths) + in_wrapped_line = prompt_len + sum(char_widths) >= self.console.width if in_wrapped_line: pos += offset - 1 # -1 cause backslash is not in buffer else: @@ -577,6 +520,7 @@ def setpos_from_xy(self, x: int, y: int) -> None: cur_x = self.screeninfo[i][0] while cur_x < x: if self.screeninfo[i][1][j] == 0: + j += 1 # prevent potential future infinite loop continue cur_x += self.screeninfo[i][1][j] j += 1 @@ -586,29 +530,33 @@ def setpos_from_xy(self, x: int, y: int) -> None: def pos2xy(self) -> tuple[int, int]: """Return the x, y coordinates of position 'pos'.""" - # this *is* incomprehensible, yes. - p, y = 0, 0 - l2: list[int] = [] + + prompt_len, y = 0, 0 + char_widths: list[int] = [] pos = self.pos assert 0 <= pos <= len(self.buffer) + + # optimize for the common case: typing at the end of the buffer if pos == len(self.buffer) and len(self.screeninfo) > 0: y = len(self.screeninfo) - 1 - p, l2 = self.screeninfo[y] - return p + sum(l2) + l2.count(0), y + prompt_len, char_widths = self.screeninfo[y] + return prompt_len + sum(char_widths), y + + for prompt_len, char_widths in self.screeninfo: + offset = len(char_widths) + in_wrapped_line = prompt_len + sum(char_widths) >= self.console.width + if in_wrapped_line: + offset -= 1 # need to remove line-wrapping backslash - for p, l2 in self.screeninfo: - l = len(l2) - l2.count(0) - in_wrapped_line = p + sum(l2) >= self.console.width - offset = l - 1 if in_wrapped_line else l # need to remove backslash if offset >= pos: break - if p + sum(l2) >= self.console.width: - pos -= l - 1 # -1 cause backslash is not in buffer - else: - pos -= l + 1 # +1 cause newline is in buffer + if not in_wrapped_line: + offset += 1 # there's a newline in buffer + + pos -= offset y += 1 - return p + sum(l2[:pos]), y + return prompt_len + sum(char_widths[:pos]), y def insert(self, text: str | list[str]) -> None: """Insert 'text' at the insertion point.""" @@ -619,6 +567,7 @@ def insert(self, text: str | list[str]) -> None: def update_cursor(self) -> None: """Move the cursor to reflect changes in self.pos""" self.cxy = self.pos2xy() + trace("update_cursor({pos}) = {cxy}", pos=self.pos, cxy=self.cxy) self.console.move_cursor(*self.cxy) def after_command(self, cmd: Command) -> None: @@ -670,6 +619,16 @@ def suspend(self) -> SimpleContextManager: setattr(self, arg, prev_state[arg]) self.prepare() + @contextmanager + def suspend_colorization(self) -> SimpleContextManager: + try: + old_can_colorize = self.can_colorize + self.can_colorize = False + yield + finally: + self.can_colorize = old_can_colorize + + def finish(self) -> None: """Called when a command signals that we're finished.""" pass @@ -685,9 +644,7 @@ def update_screen(self) -> None: def refresh(self) -> None: """Recalculate and refresh the screen.""" - if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n": - return - + self.console.height, self.console.width = self.console.getheightwidth() # this call sets up self.cxy, so call it first. self.screen = self.calc_screen() self.console.refresh(self.screen, self.cxy) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 888185eb03b..23b8fa6b9c7 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -32,20 +32,22 @@ from dataclasses import dataclass, field import os -from site import gethistoryfile # type: ignore[attr-defined] +from site import gethistoryfile import sys from rlcompleter import Completer as RLCompleter from . import commands, historical_reader from .completing_reader import CompletingReader from .console import Console as ConsoleType +from ._module_completer import ModuleCompleter, make_default_module_completer Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] -try: - from .unix_console import UnixConsole as Console, _error -except ImportError: + +if os.name == "nt": from .windows_console import WindowsConsole as Console, _error +else: + from .unix_console import UnixConsole as Console, _error ENCODING = sys.getdefaultencoding() or "latin1" @@ -89,6 +91,7 @@ # "set_pre_input_hook", "set_startup_hook", "write_history_file", + "append_history_file", # ---- multiline extensions ---- "multiline_input", ] @@ -99,7 +102,7 @@ class ReadlineConfig: readline_completer: Completer | None = None completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") - + module_completer: ModuleCompleter = field(default_factory=make_default_module_completer) @dataclass(kw_only=True) class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): @@ -132,6 +135,9 @@ def get_stem(self) -> str: return "".join(b[p + 1 : self.pos]) def get_completions(self, stem: str) -> list[str]: + module_completions = self.get_module_completions() + if module_completions is not None: + return module_completions if len(stem) == 0 and self.more_lines is not None: b = self.buffer p = self.pos @@ -161,6 +167,10 @@ def get_completions(self, stem: str) -> list[str]: result.sort() return result + def get_module_completions(self) -> list[str] | None: + line = self.get_line() + return self.config.module_completer.get_completions(line) + def get_trimmed_history(self, maxlength: int) -> list[str]: if maxlength >= 0: cut = len(self.history) - maxlength @@ -268,10 +278,6 @@ def do(self) -> None: r = self.reader # type: ignore[assignment] r.dirty = True # this is needed to hide the completion menu, if visible - if self.reader.in_bracketed_paste: - r.insert("\n") - return - # if there are already several lines and the cursor # is not on the last one, always insert a new \n. text = r.get_unicode() @@ -446,6 +452,7 @@ def read_history_file(self, filename: str = gethistoryfile()) -> None: del buffer[:] if line: history.append(line) + self.set_history_length(self.get_current_history_length()) def write_history_file(self, filename: str = gethistoryfile()) -> None: maxlength = self.saved_history_length @@ -457,6 +464,19 @@ def write_history_file(self, filename: str = gethistoryfile()) -> None: entry = entry.replace("\n", "\r\n") # multiline history support f.write(entry + "\n") + def append_history_file(self, filename: str = gethistoryfile()) -> None: + reader = self.get_reader() + saved_length = self.get_history_length() + length = self.get_current_history_length() - saved_length + history = reader.get_trimmed_history(length) + f = open(os.path.expanduser(filename), "a", + encoding="utf-8", newline="\n") + with f: + for entry in history: + entry = entry.replace("\n", "\r\n") # multiline history support + f.write(entry + "\n") + self.set_history_length(saved_length + length) + def clear_history(self) -> None: del self.get_reader().history[:] @@ -526,6 +546,7 @@ def insert_text(self, text: str) -> None: get_current_history_length = _wrapper.get_current_history_length read_history_file = _wrapper.read_history_file write_history_file = _wrapper.write_history_file +append_history_file = _wrapper.append_history_file clear_history = _wrapper.clear_history get_history_item = _wrapper.get_history_item remove_history_item = _wrapper.remove_history_item @@ -587,6 +608,7 @@ def _setup(namespace: Mapping[str, Any]) -> None: # set up namespace in rlcompleter, which requires it to be a bona fide dict if not isinstance(namespace, dict): namespace = dict(namespace) + _wrapper.config.module_completer = ModuleCompleter(namespace) _wrapper.config.readline_completer = RLCompleter(namespace).complete # this is not really what readline.c does. Better than nothing I guess diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 66e66eae7ea..6508f0233b9 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -26,18 +26,14 @@ from __future__ import annotations import _sitebuiltins -import linecache import functools import os import sys import code +import warnings +import errno -from .readline import _get_reader, multiline_input - -TYPE_CHECKING = False - -if TYPE_CHECKING: - from typing import Any +from .readline import _get_reader, multiline_input, append_history_file _error: tuple[type[Exception], ...] | type[Exception] @@ -115,6 +111,10 @@ def run_multiline_interactive_console( more_lines = functools.partial(_more_lines, console) input_n = 0 + _is_x_showrefcount_set = sys._xoptions.get("showrefcount") + _is_pydebug_build = hasattr(sys, "gettotalrefcount") + show_ref_count = _is_x_showrefcount_set and _is_pydebug_build + def maybe_run_command(statement: str) -> bool: statement = statement.strip() if statement in console.locals or statement not in REPL_COMMANDS: @@ -125,12 +125,12 @@ def maybe_run_command(statement: str) -> bool: command = REPL_COMMANDS[statement] if callable(command): # Make sure that history does not change because of commands - with reader.suspend_history(): + with reader.suspend_history(), reader.suspend_colorization(): command() return True return False - while 1: + while True: try: try: sys.stdout.flush() @@ -148,20 +148,34 @@ def maybe_run_command(statement: str) -> bool: continue input_name = f"" - linecache._register_code(input_name, statement, "") # type: ignore[attr-defined] more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single") # type: ignore[call-arg] assert not more + try: + append_history_file() + except (FileNotFoundError, PermissionError, OSError) as e: + warnings.warn(f"failed to open the history file for writing: {e}") + input_n += 1 except KeyboardInterrupt: r = _get_reader() + r.cmpltn_reset() if r.input_trans is r.isearch_trans: r.do_cmd(("isearch-end", [""])) r.pos = len(r.get_unicode()) r.dirty = True r.refresh() - r.in_bracketed_paste = False console.write("\nKeyboardInterrupt\n") console.resetbuffer() except MemoryError: console.write("\nMemoryError\n") console.resetbuffer() + except SystemExit: + raise + except: + console.showtraceback() + console.resetbuffer() + if show_ref_count: + console.write( + f"[{sys.gettotalrefcount()} refs," + f" {sys.getallocatedblocks()} blocks]\n" + ) diff --git a/Lib/_pyrepl/terminfo.py b/Lib/_pyrepl/terminfo.py new file mode 100644 index 00000000000..d02ef69cce0 --- /dev/null +++ b/Lib/_pyrepl/terminfo.py @@ -0,0 +1,488 @@ +"""Pure Python curses-like terminal capability queries.""" + +from dataclasses import dataclass, field +import errno +import os +from pathlib import Path +import re +import struct + + +# Terminfo constants +MAGIC16 = 0o432 # Magic number for 16-bit terminfo format +MAGIC32 = 0o1036 # Magic number for 32-bit terminfo format + +# Special values for absent/cancelled capabilities +ABSENT_BOOLEAN = -1 +ABSENT_NUMERIC = -1 +CANCELLED_NUMERIC = -2 +ABSENT_STRING = None +CANCELLED_STRING = None + + +# Standard string capability names from ncurses Caps file +# This matches the order used by ncurses when compiling terminfo +# fmt: off +_STRING_NAMES: tuple[str, ...] = ( + "cbt", "bel", "cr", "csr", "tbc", "clear", "el", "ed", "hpa", "cmdch", + "cup", "cud1", "home", "civis", "cub1", "mrcup", "cnorm", "cuf1", "ll", + "cuu1", "cvvis", "dch1", "dl1", "dsl", "hd", "smacs", "blink", "bold", + "smcup", "smdc", "dim", "smir", "invis", "prot", "rev", "smso", "smul", + "ech", "rmacs", "sgr0", "rmcup", "rmdc", "rmir", "rmso", "rmul", "flash", + "ff", "fsl", "is1", "is2", "is3", "if", "ich1", "il1", "ip", "kbs", "ktbc", + "kclr", "kctab", "kdch1", "kdl1", "kcud1", "krmir", "kel", "ked", "kf0", + "kf1", "kf10", "kf2", "kf3", "kf4", "kf5", "kf6", "kf7", "kf8", "kf9", + "khome", "kich1", "kil1", "kcub1", "kll", "knp", "kpp", "kcuf1", "kind", + "kri", "khts", "kcuu1", "rmkx", "smkx", "lf0", "lf1", "lf10", "lf2", "lf3", + "lf4", "lf5", "lf6", "lf7", "lf8", "lf9", "rmm", "smm", "nel", "pad", "dch", + "dl", "cud", "ich", "indn", "il", "cub", "cuf", "rin", "cuu", "pfkey", + "pfloc", "pfx", "mc0", "mc4", "mc5", "rep", "rs1", "rs2", "rs3", "rf", "rc", + "vpa", "sc", "ind", "ri", "sgr", "hts", "wind", "ht", "tsl", "uc", "hu", + "iprog", "ka1", "ka3", "kb2", "kc1", "kc3", "mc5p", "rmp", "acsc", "pln", + "kcbt", "smxon", "rmxon", "smam", "rmam", "xonc", "xoffc", "enacs", "smln", + "rmln", "kbeg", "kcan", "kclo", "kcmd", "kcpy", "kcrt", "kend", "kent", + "kext", "kfnd", "khlp", "kmrk", "kmsg", "kmov", "knxt", "kopn", "kopt", + "kprv", "kprt", "krdo", "kref", "krfr", "krpl", "krst", "kres", "ksav", + "kspd", "kund", "kBEG", "kCAN", "kCMD", "kCPY", "kCRT", "kDC", "kDL", + "kslt", "kEND", "kEOL", "kEXT", "kFND", "kHLP", "kHOM", "kIC", "kLFT", + "kMSG", "kMOV", "kNXT", "kOPT", "kPRV", "kPRT", "kRDO", "kRPL", "kRIT", + "kRES", "kSAV", "kSPD", "kUND", "rfi", "kf11", "kf12", "kf13", "kf14", + "kf15", "kf16", "kf17", "kf18", "kf19", "kf20", "kf21", "kf22", "kf23", + "kf24", "kf25", "kf26", "kf27", "kf28", "kf29", "kf30", "kf31", "kf32", + "kf33", "kf34", "kf35", "kf36", "kf37", "kf38", "kf39", "kf40", "kf41", + "kf42", "kf43", "kf44", "kf45", "kf46", "kf47", "kf48", "kf49", "kf50", + "kf51", "kf52", "kf53", "kf54", "kf55", "kf56", "kf57", "kf58", "kf59", + "kf60", "kf61", "kf62", "kf63", "el1", "mgc", "smgl", "smgr", "fln", "sclk", + "dclk", "rmclk", "cwin", "wingo", "hup","dial", "qdial", "tone", "pulse", + "hook", "pause", "wait", "u0", "u1", "u2", "u3", "u4", "u5", "u6", "u7", + "u8", "u9", "op", "oc", "initc", "initp", "scp", "setf", "setb", "cpi", + "lpi", "chr", "cvr", "defc", "swidm", "sdrfq", "sitm", "slm", "smicm", + "snlq", "snrmq", "sshm", "ssubm", "ssupm", "sum", "rwidm", "ritm", "rlm", + "rmicm", "rshm", "rsubm", "rsupm", "rum", "mhpa", "mcud1", "mcub1", "mcuf1", + "mvpa", "mcuu1", "porder", "mcud", "mcub", "mcuf", "mcuu", "scs", "smgb", + "smgbp", "smglp", "smgrp", "smgt", "smgtp", "sbim", "scsd", "rbim", "rcsd", + "subcs", "supcs", "docr", "zerom", "csnm", "kmous", "minfo", "reqmp", + "getm", "setaf", "setab", "pfxl", "devt", "csin", "s0ds", "s1ds", "s2ds", + "s3ds", "smglr", "smgtb", "birep", "binel", "bicr", "colornm", "defbi", + "endbi", "setcolor", "slines", "dispc", "smpch", "rmpch", "smsc", "rmsc", + "pctrm", "scesc", "scesa", "ehhlm", "elhlm", "elohlm", "erhlm", "ethlm", + "evhlm", "sgr1", "slength", "OTi2", "OTrs", "OTnl", "OTbc", "OTko", "OTma", + "OTG2", "OTG3", "OTG1", "OTG4", "OTGR", "OTGL", "OTGU", "OTGD", "OTGH", + "OTGV", "OTGC","meml", "memu", "box1" +) +# fmt: on + + +def _get_terminfo_dirs() -> list[Path]: + """Get list of directories to search for terminfo files. + + Based on ncurses behavior in: + - ncurses/tinfo/db_iterator.c:_nc_next_db() + - ncurses/tinfo/read_entry.c:_nc_read_entry() + """ + dirs = [] + + terminfo = os.environ.get("TERMINFO") + if terminfo: + dirs.append(terminfo) + + try: + home = Path.home() + dirs.append(str(home / ".terminfo")) + except RuntimeError: + pass + + # Check TERMINFO_DIRS + terminfo_dirs = os.environ.get("TERMINFO_DIRS", "") + if terminfo_dirs: + for d in terminfo_dirs.split(":"): + if d: + dirs.append(d) + + dirs.extend( + [ + "/etc/terminfo", + "/lib/terminfo", + "/usr/lib/terminfo", + "/usr/share/terminfo", + "/usr/share/lib/terminfo", + "/usr/share/misc/terminfo", + "/usr/local/lib/terminfo", + "/usr/local/share/terminfo", + ] + ) + + return [Path(d) for d in dirs if Path(d).is_dir()] + + +def _validate_terminal_name_or_raise(terminal_name: str) -> None: + if not isinstance(terminal_name, str): + raise TypeError("`terminal_name` must be a string") + + if not terminal_name: + raise ValueError("`terminal_name` cannot be empty") + + if "\x00" in terminal_name: + raise ValueError("NUL character found in `terminal_name`") + + t = Path(terminal_name) + if len(t.parts) > 1: + raise ValueError("`terminal_name` cannot contain path separators") + + +def _read_terminfo_file(terminal_name: str) -> bytes: + """Find and read terminfo file for given terminal name. + + Terminfo files are stored in directories using the first character + of the terminal name as a subdirectory. + """ + _validate_terminal_name_or_raise(terminal_name) + first_char = terminal_name[0].lower() + filename = terminal_name + + for directory in _get_terminfo_dirs(): + path = directory / first_char / filename + if path.is_file(): + return path.read_bytes() + + # Try with hex encoding of first char (for special chars) + hex_dir = "%02x" % ord(first_char) + path = directory / hex_dir / filename + if path.is_file(): + return path.read_bytes() + + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), filename) + + +# Hard-coded terminal capabilities for common terminals +# This is a minimal subset needed by PyREPL +_TERMINAL_CAPABILITIES = { + # ANSI/xterm-compatible terminals + "ansi": { + # Bell + "bel": b"\x07", + # Cursor movement + "cub": b"\x1b[%p1%dD", # Move cursor left N columns + "cud": b"\x1b[%p1%dB", # Move cursor down N rows + "cuf": b"\x1b[%p1%dC", # Move cursor right N columns + "cuu": b"\x1b[%p1%dA", # Move cursor up N rows + "cub1": b"\x08", # Move cursor left 1 column + "cud1": b"\n", # Move cursor down 1 row + "cuf1": b"\x1b[C", # Move cursor right 1 column + "cuu1": b"\x1b[A", # Move cursor up 1 row + "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column + "hpa": b"\x1b[%i%p1%dG", # Move cursor to column + # Clear operations + "clear": b"\x1b[H\x1b[2J", # Clear screen and home cursor + "el": b"\x1b[K", # Clear to end of line + # Insert/delete + "dch": b"\x1b[%p1%dP", # Delete N characters + "dch1": b"\x1b[P", # Delete 1 character + "ich": b"\x1b[%p1%d@", # Insert N characters + "ich1": b"", # Insert 1 character + # Cursor visibility + "civis": b"\x1b[?25l", # Make cursor invisible + "cnorm": b"\x1b[?12l\x1b[?25h", # Make cursor normal (visible) + # Scrolling + "ind": b"\n", # Scroll up one line + "ri": b"\x1bM", # Scroll down one line + # Keypad mode + "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode + "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode + # Padding (not used in modern terminals) + "pad": b"", + # Function keys and special keys + "kdch1": b"\x1b[3~", # Delete key + "kcud1": b"\x1bOB", # Down arrow + "kend": b"\x1bOF", # End key + "kent": b"\x1bOM", # Enter key + "khome": b"\x1bOH", # Home key + "kich1": b"\x1b[2~", # Insert key + "kcub1": b"\x1bOD", # Left arrow + "knp": b"\x1b[6~", # Page down + "kpp": b"\x1b[5~", # Page up + "kcuf1": b"\x1bOC", # Right arrow + "kcuu1": b"\x1bOA", # Up arrow + # Function keys F1-F20 + "kf1": b"\x1bOP", + "kf2": b"\x1bOQ", + "kf3": b"\x1bOR", + "kf4": b"\x1bOS", + "kf5": b"\x1b[15~", + "kf6": b"\x1b[17~", + "kf7": b"\x1b[18~", + "kf8": b"\x1b[19~", + "kf9": b"\x1b[20~", + "kf10": b"\x1b[21~", + "kf11": b"\x1b[23~", + "kf12": b"\x1b[24~", + "kf13": b"\x1b[1;2P", + "kf14": b"\x1b[1;2Q", + "kf15": b"\x1b[1;2R", + "kf16": b"\x1b[1;2S", + "kf17": b"\x1b[15;2~", + "kf18": b"\x1b[17;2~", + "kf19": b"\x1b[18;2~", + "kf20": b"\x1b[19;2~", + }, + # Dumb terminal - minimal capabilities + "dumb": { + "bel": b"\x07", # Bell + "cud1": b"\n", # Move down 1 row (newline) + "ind": b"\n", # Scroll up one line (newline) + }, + # Linux console + "linux": { + # Bell + "bel": b"\x07", + # Cursor movement + "cub": b"\x1b[%p1%dD", # Move cursor left N columns + "cud": b"\x1b[%p1%dB", # Move cursor down N rows + "cuf": b"\x1b[%p1%dC", # Move cursor right N columns + "cuu": b"\x1b[%p1%dA", # Move cursor up N rows + "cub1": b"\x08", # Move cursor left 1 column (backspace) + "cud1": b"\n", # Move cursor down 1 row (newline) + "cuf1": b"\x1b[C", # Move cursor right 1 column + "cuu1": b"\x1b[A", # Move cursor up 1 row + "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column + "hpa": b"\x1b[%i%p1%dG", # Move cursor to column + # Clear operations + "clear": b"\x1b[H\x1b[J", # Clear screen and home cursor (different from ansi!) + "el": b"\x1b[K", # Clear to end of line + # Insert/delete + "dch": b"\x1b[%p1%dP", # Delete N characters + "dch1": b"\x1b[P", # Delete 1 character + "ich": b"\x1b[%p1%d@", # Insert N characters + "ich1": b"\x1b[@", # Insert 1 character + # Cursor visibility + "civis": b"\x1b[?25l\x1b[?1c", # Make cursor invisible + "cnorm": b"\x1b[?25h\x1b[?0c", # Make cursor normal + # Scrolling + "ind": b"\n", # Scroll up one line + "ri": b"\x1bM", # Scroll down one line + # Keypad mode + "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode + "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode + # Function keys and special keys + "kdch1": b"\x1b[3~", # Delete key + "kcud1": b"\x1b[B", # Down arrow + "kend": b"\x1b[4~", # End key (different from ansi!) + "khome": b"\x1b[1~", # Home key (different from ansi!) + "kich1": b"\x1b[2~", # Insert key + "kcub1": b"\x1b[D", # Left arrow + "knp": b"\x1b[6~", # Page down + "kpp": b"\x1b[5~", # Page up + "kcuf1": b"\x1b[C", # Right arrow + "kcuu1": b"\x1b[A", # Up arrow + # Function keys + "kf1": b"\x1b[[A", + "kf2": b"\x1b[[B", + "kf3": b"\x1b[[C", + "kf4": b"\x1b[[D", + "kf5": b"\x1b[[E", + "kf6": b"\x1b[17~", + "kf7": b"\x1b[18~", + "kf8": b"\x1b[19~", + "kf9": b"\x1b[20~", + "kf10": b"\x1b[21~", + "kf11": b"\x1b[23~", + "kf12": b"\x1b[24~", + "kf13": b"\x1b[25~", + "kf14": b"\x1b[26~", + "kf15": b"\x1b[28~", + "kf16": b"\x1b[29~", + "kf17": b"\x1b[31~", + "kf18": b"\x1b[32~", + "kf19": b"\x1b[33~", + "kf20": b"\x1b[34~", + }, +} + +# Map common TERM values to capability sets +_TERM_ALIASES = { + "xterm": "ansi", + "xterm-color": "ansi", + "xterm-256color": "ansi", + "screen": "ansi", + "screen-256color": "ansi", + "tmux": "ansi", + "tmux-256color": "ansi", + "vt100": "ansi", + "vt220": "ansi", + "rxvt": "ansi", + "rxvt-unicode": "ansi", + "rxvt-unicode-256color": "ansi", + "unknown": "dumb", +} + + +@dataclass +class TermInfo: + terminal_name: str | bytes | None + fallback: bool = True + + _capabilities: dict[str, bytes] = field(default_factory=dict) + + def __post_init__(self) -> None: + """Initialize terminal capabilities for the given terminal type. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_setup.c:setupterm() and _nc_setupterm() + - ncurses/tinfo/lib_setup.c:TINFO_SETUP_TERM() + + This version first attempts to read terminfo database files like ncurses, + then, if `fallback` is True, falls back to hardcoded capabilities for + common terminal types. + """ + # If termstr is None or empty, try to get from environment + if not self.terminal_name: + self.terminal_name = os.environ.get("TERM") or "ANSI" + + if isinstance(self.terminal_name, bytes): + self.terminal_name = self.terminal_name.decode("ascii") + + try: + self._parse_terminfo_file(self.terminal_name) + except (OSError, ValueError): + if not self.fallback: + raise + + term_type = _TERM_ALIASES.get( + self.terminal_name, self.terminal_name + ) + if term_type not in _TERMINAL_CAPABILITIES: + term_type = "dumb" + self._capabilities = _TERMINAL_CAPABILITIES[term_type].copy() + + def _parse_terminfo_file(self, terminal_name: str) -> None: + """Parse a terminfo file. + + Populate the _capabilities dict for easy retrieval + + Based on ncurses implementation in: + - ncurses/tinfo/read_entry.c:_nc_read_termtype() + - ncurses/tinfo/read_entry.c:_nc_read_file_entry() + - ncurses/tinfo/lib_ti.c:tigetstr() + """ + data = _read_terminfo_file(terminal_name) + too_short = f"TermInfo file for {terminal_name!r} too short" + offset = 12 + if len(data) < offset: + raise ValueError(too_short) + + magic, name_size, bool_count, num_count, str_count, str_size = ( + struct.unpack(" len(data): + raise ValueError(too_short) + + # Read string offsets + end_offset = offset + 2 * str_count + if offset > len(data): + raise ValueError(too_short) + string_offset_data = data[offset:end_offset] + string_offsets = [ + off for [off] in struct.iter_unpack(" len(data): + raise ValueError(too_short) + string_table = data[offset : offset + str_size] + + # Extract strings from string table + capabilities = {} + for cap, off in zip(_STRING_NAMES, string_offsets): + if off < 0: + # CANCELLED_STRING; we do not store those + continue + elif off < len(string_table): + # Find null terminator + end = string_table.find(0, off) + if end >= 0: + capabilities[cap] = string_table[off:end] + # in other cases this is ABSENT_STRING; we don't store those. + + # Note: we don't support extended capabilities since PyREPL doesn't + # need them. + + self._capabilities = capabilities + + def get(self, cap: str) -> bytes | None: + """Get terminal capability string by name. + """ + if not isinstance(cap, str): + raise TypeError(f"`cap` must be a string, not {type(cap)}") + + return self._capabilities.get(cap) + + +def tparm(cap_bytes: bytes, *params: int) -> bytes: + """Parameterize a terminal capability string. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_tparm.c:tparm() + - ncurses/tinfo/lib_tparm.c:tparam_internal() + + The ncurses version implements a full stack-based interpreter for + terminfo parameter strings. This pure Python version implements only + the subset of parameter substitution operations needed by PyREPL: + - %i (increment parameters for 1-based indexing) + - %p[1-9]%d (parameter substitution) + - %p[1-9]%{n}%+%d (parameter plus constant) + """ + if not isinstance(cap_bytes, bytes): + raise TypeError(f"`cap` must be bytes, not {type(cap_bytes)}") + + result = cap_bytes + + # %i - increment parameters (1-based instead of 0-based) + increment = b"%i" in result + if increment: + result = result.replace(b"%i", b"") + + # Replace %p1%d, %p2%d, etc. with actual parameter values + for i in range(len(params)): + pattern = b"%%p%d%%d" % (i + 1) + if pattern in result: + value = params[i] + if increment: + value += 1 + result = result.replace(pattern, str(value).encode("ascii")) + + # Handle %p1%{1}%+%d (parameter plus constant) + # Used in some cursor positioning sequences + pattern_re = re.compile(rb"%p(\d)%\{(\d+)\}%\+%d") + matches = list(pattern_re.finditer(result)) + for match in reversed(matches): # reversed to maintain positions + param_idx = int(match.group(1)) + constant = int(match.group(2)) + value = params[param_idx] + constant + result = ( + result[: match.start()] + + str(value).encode("ascii") + + result[match.end() :] + ) + + return result diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py index a8eb2433cd3..943ee12f964 100644 --- a/Lib/_pyrepl/trace.py +++ b/Lib/_pyrepl/trace.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import sys # types if False: @@ -12,10 +13,22 @@ trace_file = open(trace_filename, "a") -def trace(line: str, *k: object, **kw: object) -> None: - if trace_file is None: - return - if k or kw: - line = line.format(*k, **kw) - trace_file.write(line + "\n") - trace_file.flush() + +if sys.platform == "emscripten": + from posix import _emscripten_log + + def trace(line: str, *k: object, **kw: object) -> None: + if "PYREPL_TRACE" not in os.environ: + return + if k or kw: + line = line.format(*k, **kw) + _emscripten_log(line) + +else: + def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py index f9d48b828c7..c5b7ebc1a40 100644 --- a/Lib/_pyrepl/types.py +++ b/Lib/_pyrepl/types.py @@ -1,8 +1,10 @@ from collections.abc import Callable, Iterator -Callback = Callable[[], object] -SimpleContextManager = Iterator[None] -KeySpec = str # like r"\C-c" -CommandName = str # like "interrupt" -EventTuple = tuple[CommandName, str] -Completer = Callable[[str, int], str | None] +type Callback = Callable[[], object] +type SimpleContextManager = Iterator[None] +type KeySpec = str # like r"\C-c" +type CommandName = str # like "interrupt" +type EventTuple = tuple[CommandName, str] +type Completer = Callable[[str, int], str | None] +type CharBuffer = list[str] +type CharWidths = list[int] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index e69c96b1159..639d16db3f8 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -29,31 +29,41 @@ import struct import termios import time +import types import platform from fcntl import ioctl -from . import curses +from . import terminfo from .console import Console, Event -from .fancy_termios import tcgetattr, tcsetattr +from .fancy_termios import tcgetattr, tcsetattr, TermState from .trace import trace from .unix_eventqueue import EventQueue from .utils import wlen +# declare posix optional to allow None assignment on other platforms +posix: types.ModuleType | None +try: + import posix +except ImportError: + posix = None TYPE_CHECKING = False # types if TYPE_CHECKING: - from typing import IO, Literal, overload + from typing import AbstractSet, IO, Literal, overload, cast else: overload = lambda func: None + cast = lambda typ, val: val class InvalidTerminal(RuntimeError): - pass + def __init__(self, message: str) -> None: + super().__init__(errno.EIO, message) -_error = (termios.error, curses.error, InvalidTerminal) +_error = (termios.error, InvalidTerminal) +_error_codes_to_ignore = frozenset([errno.EIO, errno.ENXIO, errno.EPERM]) SIGWINCH_EVENT = "repaint" @@ -110,7 +120,7 @@ def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None: try: poll: type[select.poll] = select.poll except AttributeError: - # this is exactly the minumum necessary to support what we + # this is exactly the minimum necessary to support what we # do with poll objects class MinimalPoll: def __init__(self): @@ -118,12 +128,13 @@ def __init__(self): def register(self, fd, flag): self.fd = fd + # note: The 'timeout' argument is received as *milliseconds* def poll(self, timeout: float | None = None) -> list[int]: if timeout is None: r, w, e = select.select([self.fd], [], []) else: - r, w, e = select.select([self.fd], [], [], timeout/1000) + r, w, e = select.select([self.fd], [], [], timeout / 1000) return r poll = MinimalPoll # type: ignore[assignment] @@ -150,19 +161,28 @@ def __init__( self.pollob = poll() self.pollob.register(self.input_fd, select.POLLIN) - self.input_buffer = b"" - self.input_buffer_pos = 0 - curses.setupterm(term or None, self.output_fd) + self.terminfo = terminfo.TermInfo(term or None) self.term = term + self.is_apple_terminal = ( + platform.system() == "Darwin" + and os.getenv("TERM_PROGRAM") == "Apple_Terminal" + ) + + try: + self.__input_fd_set(tcgetattr(self.input_fd), ignore=frozenset()) + except _error as e: + raise RuntimeError(f"termios failure ({e.args[1]})") @overload - def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... + def _my_getstr( + cap: str, optional: Literal[False] = False + ) -> bytes: ... @overload def _my_getstr(cap: str, optional: bool) -> bytes | None: ... def _my_getstr(cap: str, optional: bool = False) -> bytes | None: - r = curses.tigetstr(cap) + r = self.terminfo.get(cap) if not optional and r is None: raise InvalidTerminal( f"terminal doesn't have the required {cap} capability" @@ -196,26 +216,19 @@ def _my_getstr(cap: str, optional: bool = False) -> bytes | None: self.__setup_movement() - self.event_queue = EventQueue(self.input_fd, self.encoding) - self.cursor_visible = 1 - - def more_in_buffer(self) -> bool: - return bool( - self.input_buffer - and self.input_buffer_pos < len(self.input_buffer) + self.event_queue = EventQueue( + self.input_fd, self.encoding, self.terminfo ) + self.cursor_visible = 1 - def __read(self, n: int) -> bytes: - if not self.more_in_buffer(): - self.input_buffer = os.read(self.input_fd, 10000) + signal.signal(signal.SIGCONT, self._sigcont_handler) - ret = self.input_buffer[self.input_buffer_pos : self.input_buffer_pos + n] - self.input_buffer_pos += len(ret) - if self.input_buffer_pos >= len(self.input_buffer): - self.input_buffer = b"" - self.input_buffer_pos = 0 - return ret + def _sigcont_handler(self, signum, frame): + self.restore() + self.prepare() + def __read(self, n: int) -> bytes: + return os.read(self.input_fd, n) def change_encoding(self, encoding: str) -> None: """ @@ -238,8 +251,9 @@ def refresh(self, screen, c_xy): if not self.__gone_tall: while len(self.screen) < min(len(screen), self.height): self.__hide_cursor() - self.__move(0, len(self.screen) - 1) - self.__write("\n") + if self.screen: + self.__move(0, len(self.screen) - 1) + self.__write("\n") self.posxy = 0, len(self.screen) self.screen.append("") else: @@ -328,6 +342,8 @@ def prepare(self): """ Prepare the console for input/output operations. """ + self.__buffer = [] + self.__svtermstate = tcgetattr(self.input_fd) raw = self.__svtermstate.copy() raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON) @@ -339,17 +355,15 @@ def prepare(self): raw.lflag |= termios.ISIG raw.cc[termios.VMIN] = 1 raw.cc[termios.VTIME] = 0 - tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + self.__input_fd_set(raw) # In macOS terminal we need to deactivate line wrap via ANSI escape code - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7l") self.screen = [] self.height, self.width = self.getheightwidth() - self.__buffer = [] - self.posxy = 0, 0 self.__gone_tall = 0 self.__move = self.__move_short @@ -371,13 +385,18 @@ def restore(self): self.__disable_bracketed_paste() self.__maybe_write_code(self._rmkx) self.flushoutput() - tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + self.__input_fd_set(self.__svtermstate) - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7h") if hasattr(self, "old_sigwinch"): - signal.signal(signal.SIGWINCH, self.old_sigwinch) + try: + signal.signal(signal.SIGWINCH, self.old_sigwinch) + except ValueError as e: + import threading + if threading.current_thread() is threading.main_thread(): + raise e del self.old_sigwinch def push_char(self, char: int | bytes) -> None: @@ -410,6 +429,8 @@ def get_event(self, block: bool = True) -> Event | None: return self.event_queue.get() else: continue + elif err.errno == errno.EIO: + raise SystemExit(errno.EIO) else: raise else: @@ -422,7 +443,6 @@ def wait(self, timeout: float | None = None) -> bool: """ return ( not self.event_queue.empty() - or self.more_in_buffer() or bool(self.pollob.poll(timeout)) ) @@ -525,6 +545,7 @@ def getpending(self): e.raw += e.raw amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] + trace("getpending({a})", a=amount) raw = self.__read(amount) data = str(raw, self.encoding, "replace") e.data += data @@ -566,11 +587,9 @@ def clear(self): @property def input_hook(self): - try: - import posix - except ImportError: - return None - if posix._is_inputhook_installed(): + # avoid inline imports here so the repl doesn't get flooded + # with import logging from -X importtime=2 + if posix is not None and posix._is_inputhook_installed(): return posix._inputhook def __enable_bracketed_paste(self) -> None: @@ -602,14 +621,14 @@ def __setup_movement(self): if self._dch1: self.dch1 = self._dch1 elif self._dch: - self.dch1 = curses.tparm(self._dch, 1) + self.dch1 = terminfo.tparm(self._dch, 1) else: self.dch1 = None if self._ich1: self.ich1 = self._ich1 elif self._ich: - self.ich1 = curses.tparm(self._ich, 1) + self.ich1 = terminfo.tparm(self._ich, 1) else: self.ich1 = None @@ -634,7 +653,7 @@ def __write_changed_line(self, y, oldline, newline, px_coord): # reuse the oldline as much as possible, but stop as soon as we # encounter an ESCAPE, because it might be the start of an escape - # sequene + # sequence while ( x_coord < minlen and oldline[x_pos] == newline[x_pos] @@ -706,7 +725,7 @@ def __write(self, text): self.__buffer.append((text, 0)) def __write_code(self, fmt, *args): - self.__buffer.append((curses.tparm(fmt, *args), 1)) + self.__buffer.append((terminfo.tparm(fmt, *args), 1)) def __maybe_write_code(self, fmt, *args): if fmt: @@ -757,7 +776,6 @@ def __move_tall(self, x, y): self.__write_code(self._cup, y - self.__offset, x) def __sigwinch(self, signum, frame): - self.height, self.width = self.getheightwidth() self.event_queue.insert(Event("resize", None)) def __hide_cursor(self): @@ -790,9 +808,9 @@ def __tputs(self, fmt, prog=delayprog): will never do anyone any good.""" # using .get() means that things will blow up # only if the bps is actually needed (which I'm - # betting is pretty unlkely) + # betting is pretty unlikely) bps = ratedict.get(self.__svtermstate.ospeed) - while 1: + while True: m = prog.search(fmt) if not m: os.write(self.output_fd, fmt) @@ -808,3 +826,17 @@ def __tputs(self, fmt, prog=delayprog): os.write(self.output_fd, self._pad * nchars) else: time.sleep(float(delay) / 1000.0) + + def __input_fd_set( + self, + state: TermState, + ignore: AbstractSet[int] = _error_codes_to_ignore, + ) -> bool: + try: + tcsetattr(self.input_fd, termios.TCSADRAIN, state) + except termios.error as te: + if te.args[0] not in ignore: + raise + return False + else: + return True diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py index 70cfade26e2..2a9cca59e74 100644 --- a/Lib/_pyrepl/unix_eventqueue.py +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -18,12 +18,9 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from collections import deque - -from . import keymap -from .console import Event -from . import curses +from .terminfo import TermInfo from .trace import trace +from .base_eventqueue import BaseEventQueue from termios import tcgetattr, VERASE import os @@ -57,96 +54,24 @@ b'\033Oc': 'ctrl right', } -def get_terminal_keycodes() -> dict[bytes, str]: +def get_terminal_keycodes(ti: TermInfo) -> dict[bytes, str]: """ Generates a dictionary mapping terminal keycodes to human-readable names. """ keycodes = {} for key, terminal_code in TERMINAL_KEYNAMES.items(): - keycode = curses.tigetstr(terminal_code) + keycode = ti.get(terminal_code) trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) if keycode: keycodes[keycode] = key keycodes.update(CTRL_ARROW_KEYCODES) return keycodes -class EventQueue: - def __init__(self, fd: int, encoding: str) -> None: - self.keycodes = get_terminal_keycodes() + +class EventQueue(BaseEventQueue): + def __init__(self, fd: int, encoding: str, ti: TermInfo) -> None: + keycodes = get_terminal_keycodes(ti) if os.isatty(fd): backspace = tcgetattr(fd)[6][VERASE] - self.keycodes[backspace] = "backspace" - self.compiled_keymap = keymap.compile_keymap(self.keycodes) - self.keymap = self.compiled_keymap - trace("keymap {k!r}", k=self.keymap) - self.encoding = encoding - self.events: deque[Event] = deque() - self.buf = bytearray() - - def get(self) -> Event | None: - """ - Retrieves the next event from the queue. - """ - if self.events: - return self.events.popleft() - else: - return None - - def empty(self) -> bool: - """ - Checks if the queue is empty. - """ - return not self.events - - def flush_buf(self) -> bytearray: - """ - Flushes the buffer and returns its contents. - """ - old = self.buf - self.buf = bytearray() - return old - - def insert(self, event: Event) -> None: - """ - Inserts an event into the queue. - """ - trace('added event {event}', event=event) - self.events.append(event) - - def push(self, char: int | bytes) -> None: - """ - Processes a character by updating the buffer and handling special key mappings. - """ - ord_char = char if isinstance(char, int) else ord(char) - char = bytes(bytearray((ord_char,))) - self.buf.append(ord_char) - if char in self.keymap: - if self.keymap is self.compiled_keymap: - #sanity check, buffer is empty when a special key comes - assert len(self.buf) == 1 - k = self.keymap[char] - trace('found map {k!r}', k=k) - if isinstance(k, dict): - self.keymap = k - else: - self.insert(Event('key', k, self.flush_buf())) - self.keymap = self.compiled_keymap - - elif self.buf and self.buf[0] == 27: # escape - # escape sequence not recognized by our keymap: propagate it - # outside so that i can be recognized as an M-... key (see also - # the docstring in keymap.py - trace('unrecognized escape sequence, propagating...') - self.keymap = self.compiled_keymap - self.insert(Event('key', '\033', bytearray(b'\033'))) - for _c in self.flush_buf()[1:]: - self.push(_c) - - else: - try: - decoded = bytes(self.buf).decode(self.encoding) - except UnicodeError: - return - else: - self.insert(Event('key', decoded, self.flush_buf())) - self.keymap = self.compiled_keymap + keycodes[backspace] = "backspace" + BaseEventQueue.__init__(self, encoding, keycodes) diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 4651717bd7e..06cddef851b 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -1,25 +1,391 @@ +from __future__ import annotations +import builtins +import functools +import keyword import re +import token as T +import tokenize import unicodedata -import functools +import _colorize + +from collections import deque +from io import StringIO +from tokenize import TokenInfo as TI +from typing import Iterable, Iterator, Match, NamedTuple, Self + +from .types import CharBuffer, CharWidths +from .trace import trace ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") +ZERO_WIDTH_BRACKET = re.compile(r"\x01.*?\x02") +ZERO_WIDTH_TRANS = str.maketrans({"\x01": "", "\x02": ""}) +IDENTIFIERS_AFTER = {"def", "class"} +KEYWORD_CONSTANTS = {"True", "False", "None"} +BUILTINS = {str(name) for name in dir(builtins) if not name.startswith('_')} + + +def THEME(**kwargs): + # Not cached: the user can modify the theme inside the interactive session. + return _colorize.get_theme(**kwargs).syntax + + +class Span(NamedTuple): + """Span indexing that's inclusive on both ends.""" + + start: int + end: int + + @classmethod + def from_re(cls, m: Match[str], group: int | str) -> Self: + re_span = m.span(group) + return cls(re_span[0], re_span[1] - 1) + + @classmethod + def from_token(cls, token: TI, line_len: list[int]) -> Self: + end_offset = -1 + if (token.type in {T.FSTRING_MIDDLE, T.TSTRING_MIDDLE} + and token.string.endswith(("{", "}"))): + # gh-134158: a visible trailing brace comes from a double brace in input + end_offset += 1 + + return cls( + line_len[token.start[0] - 1] + token.start[1], + line_len[token.end[0] - 1] + token.end[1] + end_offset, + ) + + +class ColorSpan(NamedTuple): + span: Span + tag: str @functools.cache def str_width(c: str) -> int: if ord(c) < 128: return 1 + # gh-139246 for zero-width joiner and combining characters + if unicodedata.combining(c): + return 0 + category = unicodedata.category(c) + if category == "Cf" and c != "\u00ad": + return 0 w = unicodedata.east_asian_width(c) - if w in ('N', 'Na', 'H', 'A'): + if w in ("N", "Na", "H", "A"): return 1 return 2 def wlen(s: str) -> int: - if len(s) == 1 and s != '\x1a': + if len(s) == 1 and s != "\x1a": return str_width(s) length = sum(str_width(i) for i in s) # remove lengths of any escape sequences sequence = ANSI_ESCAPE_SEQUENCE.findall(s) - ctrl_z_cnt = s.count('\x1a') + ctrl_z_cnt = s.count("\x1a") return length - sum(len(i) for i in sequence) + ctrl_z_cnt + + +def unbracket(s: str, including_content: bool = False) -> str: + r"""Return `s` with \001 and \002 characters removed. + + If `including_content` is True, content between \001 and \002 is also + stripped. + """ + if including_content: + return ZERO_WIDTH_BRACKET.sub("", s) + return s.translate(ZERO_WIDTH_TRANS) + + +def gen_colors(buffer: str) -> Iterator[ColorSpan]: + """Returns a list of index spans to color using the given color tag. + + The input `buffer` should be a valid start of a Python code block, i.e. + it cannot be a block starting in the middle of a multiline string. + """ + sio = StringIO(buffer) + line_lengths = [0] + [len(line) for line in sio.readlines()] + # make line_lengths cumulative + for i in range(1, len(line_lengths)): + line_lengths[i] += line_lengths[i-1] + + sio.seek(0) + gen = tokenize.generate_tokens(sio.readline) + last_emitted: ColorSpan | None = None + try: + for color in gen_colors_from_token_stream(gen, line_lengths): + yield color + last_emitted = color + except SyntaxError: + return + except tokenize.TokenError as te: + yield from recover_unterminated_string( + te, line_lengths, last_emitted, buffer + ) + + +def recover_unterminated_string( + exc: tokenize.TokenError, + line_lengths: list[int], + last_emitted: ColorSpan | None, + buffer: str, +) -> Iterator[ColorSpan]: + msg, loc = exc.args + if loc is None: + return + + line_no, column = loc + + if msg.startswith( + ( + "unterminated string literal", + "unterminated f-string literal", + "unterminated t-string literal", + "EOF in multi-line string", + "unterminated triple-quoted f-string literal", + "unterminated triple-quoted t-string literal", + ) + ): + start = line_lengths[line_no - 1] + column - 1 + end = line_lengths[-1] - 1 + + # in case FSTRING_START was already emitted + if last_emitted and start <= last_emitted.span.start: + trace("before last emitted = {s}", s=start) + start = last_emitted.span.end + 1 + + span = Span(start, end) + trace("yielding span {a} -> {b}", a=span.start, b=span.end) + yield ColorSpan(span, "string") + else: + trace( + "unhandled token error({buffer}) = {te}", + buffer=repr(buffer), + te=str(exc), + ) + + +def gen_colors_from_token_stream( + token_generator: Iterator[TI], + line_lengths: list[int], +) -> Iterator[ColorSpan]: + token_window = prev_next_window(token_generator) + + is_def_name = False + bracket_level = 0 + for prev_token, token, next_token in token_window: + assert token is not None + if token.start == token.end: + continue + + match token.type: + case ( + T.STRING + | T.FSTRING_START | T.FSTRING_MIDDLE | T.FSTRING_END + | T.TSTRING_START | T.TSTRING_MIDDLE | T.TSTRING_END + ): + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "string") + case T.COMMENT: + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "comment") + case T.NUMBER: + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "number") + case T.OP: + if token.string in "([{": + bracket_level += 1 + elif token.string in ")]}": + bracket_level -= 1 + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "op") + case T.NAME: + if is_def_name: + is_def_name = False + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "definition") + elif keyword.iskeyword(token.string): + span_cls = "keyword" + if token.string in KEYWORD_CONSTANTS: + span_cls = "keyword_constant" + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, span_cls) + if token.string in IDENTIFIERS_AFTER: + is_def_name = True + elif ( + keyword.issoftkeyword(token.string) + and bracket_level == 0 + and is_soft_keyword_used(prev_token, token, next_token) + ): + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "soft_keyword") + elif ( + token.string in BUILTINS + and not (prev_token and prev_token.exact_type == T.DOT) + ): + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "builtin") + + +keyword_first_sets_match = {"False", "None", "True", "await", "lambda", "not"} +keyword_first_sets_case = {"False", "None", "True"} + + +def is_soft_keyword_used(*tokens: TI | None) -> bool: + """Returns True if the current token is a keyword in this context. + + For the `*tokens` to match anything, they have to be a three-tuple of + (previous, current, next). + """ + trace("is_soft_keyword_used{t}", t=tokens) + match tokens: + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + TI(string="match"), + TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) + | TI(T.OP, string="(" | "*" | "[" | "{" | "~" | "...") + ): + return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + TI(string="match"), + TI(T.NAME, string=s) + ): + if keyword.iskeyword(s): + return s in keyword_first_sets_match + return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), + TI(string="case"), + TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) + | TI(T.OP, string="(" | "*" | "-" | "[" | "{") + ): + return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), + TI(string="case"), + TI(T.NAME, string=s) + ): + if keyword.iskeyword(s): + return s in keyword_first_sets_case + return True + case (TI(string="case"), TI(string="_"), TI(string=":")): + return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), + TI(string="type"), + TI(T.NAME, string=s) + ): + return not keyword.iskeyword(s) + case _: + return False + + +def disp_str( + buffer: str, + colors: list[ColorSpan] | None = None, + start_index: int = 0, + force_color: bool = False, +) -> tuple[CharBuffer, CharWidths]: + r"""Decompose the input buffer into a printable variant with applied colors. + + Returns a tuple of two lists: + - the first list is the input buffer, character by character, with color + escape codes added (while those codes contain multiple ASCII characters, + each code is considered atomic *and is attached for the corresponding + visible character*); + - the second list is the visible width of each character in the input + buffer. + + Note on colors: + - The `colors` list, if provided, is partially consumed within. We're using + a list and not a generator since we need to hold onto the current + unfinished span between calls to disp_str in case of multiline strings. + - The `colors` list is computed from the start of the input block. `buffer` + is only a subset of that input block, a single line within. This is why + we need `start_index` to inform us which position is the start of `buffer` + actually within user input. This allows us to match color spans correctly. + + Examples: + >>> utils.disp_str("a = 9") + (['a', ' ', '=', ' ', '9'], [1, 1, 1, 1, 1]) + + >>> line = "while 1:" + >>> colors = list(utils.gen_colors(line)) + >>> utils.disp_str(line, colors=colors) + (['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1]) + + """ + chars: CharBuffer = [] + char_widths: CharWidths = [] + + if not buffer: + return chars, char_widths + + while colors and colors[0].span.end < start_index: + # move past irrelevant spans + colors.pop(0) + + theme = THEME(force_color=force_color) + pre_color = "" + post_color = "" + if colors and colors[0].span.start < start_index: + # looks like we're continuing a previous color (e.g. a multiline str) + pre_color = theme[colors[0].tag] + + for i, c in enumerate(buffer, start_index): + if colors and colors[0].span.start == i: # new color starts now + pre_color = theme[colors[0].tag] + + if c == "\x1a": # CTRL-Z on Windows + chars.append(c) + char_widths.append(2) + elif ord(c) < 128: + chars.append(c) + char_widths.append(1) + elif unicodedata.category(c).startswith("C"): + c = r"\u%04x" % ord(c) + chars.append(c) + char_widths.append(len(c)) + else: + chars.append(c) + char_widths.append(str_width(c)) + + if colors and colors[0].span.end == i: # current color ends now + post_color = theme.reset + colors.pop(0) + + chars[-1] = pre_color + chars[-1] + post_color + pre_color = "" + post_color = "" + + if colors and colors[0].span.start < i and colors[0].span.end > i: + # even though the current color should be continued, reset it for now. + # the next call to `disp_str()` will revive it. + chars[-1] += theme.reset + + return chars, char_widths + + +def prev_next_window[T]( + iterable: Iterable[T] +) -> Iterator[tuple[T | None, ...]]: + """Generates three-tuples of (previous, current, next) items. + + On the first iteration previous is None. On the last iteration next + is None. In case of exception next is None and the exception is re-raised + on a subsequent next() call. + + Inspired by `sliding_window` from `itertools` recipes. + """ + + iterator = iter(iterable) + window = deque((None, next(iterator)), maxlen=3) + try: + for x in iterator: + window.append(x) + yield tuple(window) + except Exception: + raise + finally: + window.append(None) + yield tuple(window) diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index fffadd5e2ec..46c6030748b 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -22,11 +22,9 @@ import io import os import sys -import time -import msvcrt -from collections import deque import ctypes +import types from ctypes.wintypes import ( _COORD, WORD, @@ -42,9 +40,10 @@ from .console import Event, Console from .trace import trace from .utils import wlen +from .windows_eventqueue import EventQueue try: - from ctypes import GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] + from ctypes import get_last_error, GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] except: # Keep MyPy happy off Windows from ctypes import CDLL as WinDLL, cdll as windll @@ -52,11 +51,20 @@ def GetLastError() -> int: return 42 + def get_last_error() -> int: + return 42 + class WinError(OSError): # type: ignore[no-redef] def __init__(self, err: int | None, descr: str | None = None) -> None: self.err = err self.descr = descr +# declare nt optional to allow None assignment on other platforms +nt: types.ModuleType | None +try: + import nt +except ImportError: + nt = None TYPE_CHECKING = False @@ -94,7 +102,9 @@ def __init__(self, err: int | None, descr: str | None = None) -> None: 0x83: "f20", # VK_F20 } -# Console escape codes: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences +# Virtual terminal output sequences +# Reference: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#output-sequences +# Check `windows_eventqueue.py` for input sequences ERASE_IN_LINE = "\x1b[K" MOVE_LEFT = "\x1b[{}D" MOVE_RIGHT = "\x1b[{}C" @@ -102,10 +112,25 @@ def __init__(self, err: int | None, descr: str | None = None) -> None: MOVE_DOWN = "\x1b[{}B" CLEAR = "\x1b[H\x1b[J" +# State of control keys: https://learn.microsoft.com/en-us/windows/console/key-event-record-str +ALT_ACTIVE = 0x01 | 0x02 +CTRL_ACTIVE = 0x04 | 0x08 + +WAIT_TIMEOUT = 0x102 +WAIT_FAILED = 0xFFFFFFFF + +# from winbase.h +INFINITE = 0xFFFFFFFF + class _error(Exception): pass +def _supports_vt(): + try: + return nt._supports_virtual_terminal() + except AttributeError: + return False class WindowsConsole(Console): def __init__( @@ -117,17 +142,29 @@ def __init__( ): super().__init__(f_in, f_out, term, encoding) + self.__vt_support = _supports_vt() + + if self.__vt_support: + trace('console supports virtual terminal') + + # Save original console modes so we can recover on cleanup. + original_input_mode = DWORD() + GetConsoleMode(InHandle, original_input_mode) + trace(f'saved original input mode 0x{original_input_mode.value:x}') + self.__original_input_mode = original_input_mode.value + SetConsoleMode( OutHandle, ENABLE_WRAP_AT_EOL_OUTPUT | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING, ) + self.screen: list[str] = [] self.width = 80 self.height = 25 self.__offset = 0 - self.event_queue: deque[Event] = deque() + self.event_queue = EventQueue(encoding) try: self.out = io._WindowsConsoleIO(self.output_fd, "w") # type: ignore[attr-defined] except ValueError: @@ -146,8 +183,9 @@ def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: while len(self.screen) < min(len(screen), self.height): self._hide_cursor() - self._move_relative(0, len(self.screen) - 1) - self.__write("\n") + if self.screen: + self._move_relative(0, len(self.screen) - 1) + self.__write("\n") self.posxy = 0, len(self.screen) self.screen.append("") @@ -204,11 +242,9 @@ def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: @property def input_hook(self): - try: - import nt - except ImportError: - return None - if nt._is_inputhook_installed(): + # avoid inline imports here so the repl doesn't get flooded + # with import logging from -X importtime=2 + if nt is not None and nt._is_inputhook_installed(): return nt._inputhook def __write_changed_line( @@ -232,7 +268,7 @@ def __write_changed_line( # reuse the oldline as much as possible, but stop as soon as we # encounter an ESCAPE, because it might be the start of an escape - # sequene + # sequence while ( x_coord < minlen and oldline[x_pos] == newline[x_pos] @@ -247,18 +283,13 @@ def __write_changed_line( self._erase_to_end() self.__write(newline[x_pos:]) - if wlen(newline) == self.width: - # If we wrapped we want to start at the next line - self._move_relative(0, y + 1) - self.posxy = 0, y + 1 - else: - self.posxy = wlen(newline), y + self.posxy = min(wlen(newline), self.width - 1), y - if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline: - # ANSI escape characters are present, so we can't assume - # anything about the position of the cursor. Moving the cursor - # to the left margin should work to get to a known position. - self.move_cursor(0, y) + if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) def _scroll( self, top: int, bottom: int, left: int | None = None, right: int | None = None @@ -291,6 +322,12 @@ def _enable_blinking(self): def _disable_blinking(self): self.__write("\x1b[?12l") + def _enable_bracketed_paste(self) -> None: + self.__write("\x1b[?2004h") + + def _disable_bracketed_paste(self) -> None: + self.__write("\x1b[?2004l") + def __write(self, text: str) -> None: if "\x1a" in text: text = ''.join(["^Z" if x == '\x1a' else x for x in text]) @@ -320,8 +357,15 @@ def prepare(self) -> None: self.__gone_tall = 0 self.__offset = 0 + if self.__vt_support: + SetConsoleMode(InHandle, self.__original_input_mode | ENABLE_VIRTUAL_TERMINAL_INPUT) + self._enable_bracketed_paste() + def restore(self) -> None: - pass + if self.__vt_support: + # Recover to original mode before running REPL + self._disable_bracketed_paste() + SetConsoleMode(InHandle, self.__original_input_mode) def _move_relative(self, x: int, y: int) -> None: """Moves relative to the current posxy""" @@ -342,7 +386,7 @@ def move_cursor(self, x: int, y: int) -> None: raise ValueError(f"Bad cursor position {x}, {y}") if y < self.__offset or y >= self.__offset + self.height: - self.event_queue.insert(0, Event("scroll", "")) + self.event_queue.insert(Event("scroll", "")) else: self._move_relative(x, y) self.posxy = x, y @@ -371,14 +415,7 @@ def _getscrollbacksize(self) -> int: return info.srWindow.Bottom # type: ignore[no-any-return] - def _read_input(self, block: bool = True) -> INPUT_RECORD | None: - if not block: - events = DWORD() - if not GetNumberOfConsoleInputEvents(InHandle, events): - raise WinError(GetLastError()) - if not events.value: - return None - + def _read_input(self) -> INPUT_RECORD | None: rec = INPUT_RECORD() read = DWORD() if not ReadConsoleInput(InHandle, rec, 1, read): @@ -386,15 +423,26 @@ def _read_input(self, block: bool = True) -> INPUT_RECORD | None: return rec + def _read_input_bulk( + self, n: int + ) -> tuple[ctypes.Array[INPUT_RECORD], int]: + rec = (n * INPUT_RECORD)() + read = DWORD() + if not ReadConsoleInput(InHandle, rec, n, read): + raise WinError(GetLastError()) + + return rec, read.value + def get_event(self, block: bool = True) -> Event | None: """Return an Event instance. Returns None if |block| is false and there is no event pending, otherwise waits for the completion of an event.""" - if self.event_queue: - return self.event_queue.pop() - while True: - rec = self._read_input(block) + if not block and not self.wait(timeout=0): + return None + + while self.event_queue.empty(): + rec = self._read_input() if rec is None: return None @@ -407,31 +455,47 @@ def get_event(self, block: bool = True) -> Event | None: continue return None - key = rec.Event.KeyEvent.uChar.UnicodeChar + key_event = rec.Event.KeyEvent + raw_key = key = key_event.uChar.UnicodeChar - if rec.Event.KeyEvent.uChar.UnicodeChar == "\r": - # Make enter make unix-like - return Event(evt="key", data="\n", raw=b"\n") - elif rec.Event.KeyEvent.wVirtualKeyCode == 8: + if key == "\r": + # Make enter unix-like + return Event(evt="key", data="\n") + elif key_event.wVirtualKeyCode == 8: # Turn backspace directly into the command - return Event( - evt="key", - data="backspace", - raw=rec.Event.KeyEvent.uChar.UnicodeChar, - ) - elif rec.Event.KeyEvent.uChar.UnicodeChar == "\x00": + key = "backspace" + elif key == "\x00": # Handle special keys like arrow keys and translate them into the appropriate command - code = VK_MAP.get(rec.Event.KeyEvent.wVirtualKeyCode) - if code: - return Event( - evt="key", data=code, raw=rec.Event.KeyEvent.uChar.UnicodeChar - ) + key = VK_MAP.get(key_event.wVirtualKeyCode) + if key: + if key_event.dwControlKeyState & CTRL_ACTIVE: + key = f"ctrl {key}" + elif key_event.dwControlKeyState & ALT_ACTIVE: + # queue the key, return the meta command + self.event_queue.insert(Event(evt="key", data=key)) + return Event(evt="key", data="\033") # keymap.py uses this for meta + return Event(evt="key", data=key) if block: continue return None - - return Event(evt="key", data=key, raw=rec.Event.KeyEvent.uChar.UnicodeChar) + elif self.__vt_support: + # If virtual terminal is enabled, scanning VT sequences + for char in raw_key.encode(self.event_queue.encoding, "replace"): + self.event_queue.push(char) + continue + + if key_event.dwControlKeyState & ALT_ACTIVE: + # Do not swallow characters that have been entered via AltGr: + # Windows internally converts AltGr to CTRL+ALT, see + # https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-vkkeyscanw + if not key_event.dwControlKeyState & CTRL_ACTIVE: + # queue the key, return the meta command + self.event_queue.insert(Event(evt="key", data=key)) + return Event(evt="key", data="\033") # keymap.py uses this for meta + + return Event(evt="key", data=key) + return self.event_queue.get() def push_char(self, char: int | bytes) -> None: """ @@ -446,7 +510,7 @@ def clear(self) -> None: """Wipe the screen""" self.__write(CLEAR) self.posxy = 0, 0 - self.screen = [""] + self.screen = [] def finish(self) -> None: """Move the cursor to the end of the display and otherwise get @@ -472,18 +536,53 @@ def forgetinput(self) -> None: def getpending(self) -> Event: """Return the characters that have been typed but not yet processed.""" - return Event("key", "", b"") + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + if e2: + e.data += e2.data + + recs, rec_count = self._read_input_bulk(1024) + for i in range(rec_count): + rec = recs[i] + # In case of a legacy console, we do not only receive a keydown + # event, but also a keyup event - and for uppercase letters + # an additional SHIFT_PRESSED event. + if rec and rec.EventType == KEY_EVENT: + key_event = rec.Event.KeyEvent + if not key_event.bKeyDown: + continue + ch = key_event.uChar.UnicodeChar + if ch == "\x00": + # ignore SHIFT_PRESSED and special keys + continue + if ch == "\r": + ch += "\n" + e.data += ch + return e - def wait(self, timeout: float | None) -> bool: + def wait_for_event(self, timeout: float | None) -> bool: """Wait for an event.""" - # Poor man's Windows select loop - start_time = time.time() - while True: - if msvcrt.kbhit(): # type: ignore[attr-defined] - return True - if timeout and time.time() - start_time > timeout / 1000: - return False - time.sleep(0.01) + if timeout is None: + timeout = INFINITE + else: + timeout = int(timeout) + ret = WaitForSingleObject(InHandle, timeout) + if ret == WAIT_FAILED: + raise WinError(get_last_error()) + elif ret == WAIT_TIMEOUT: + return False + return True + + def wait(self, timeout: float | None) -> bool: + """ + Wait for events on the console. + """ + return ( + not self.event_queue.empty() + or self.wait_for_event(timeout) + ) def repaint(self) -> None: raise NotImplementedError("No repaint support") @@ -553,6 +652,13 @@ class INPUT_RECORD(Structure): MOUSE_EVENT = 0x02 WINDOW_BUFFER_SIZE_EVENT = 0x04 +ENABLE_PROCESSED_INPUT = 0x0001 +ENABLE_LINE_INPUT = 0x0002 +ENABLE_ECHO_INPUT = 0x0004 +ENABLE_MOUSE_INPUT = 0x0010 +ENABLE_INSERT_MODE = 0x0020 +ENABLE_VIRTUAL_TERMINAL_INPUT = 0x0200 + ENABLE_PROCESSED_OUTPUT = 0x01 ENABLE_WRAP_AT_EOL_OUTPUT = 0x02 ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04 @@ -584,6 +690,10 @@ class INPUT_RECORD(Structure): ] ScrollConsoleScreenBuffer.restype = BOOL + GetConsoleMode = _KERNEL32.GetConsoleMode + GetConsoleMode.argtypes = [HANDLE, POINTER(DWORD)] + GetConsoleMode.restype = BOOL + SetConsoleMode = _KERNEL32.SetConsoleMode SetConsoleMode.argtypes = [HANDLE, DWORD] SetConsoleMode.restype = BOOL @@ -592,14 +702,15 @@ class INPUT_RECORD(Structure): ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)] ReadConsoleInput.restype = BOOL - GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents - GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)] - GetNumberOfConsoleInputEvents.restype = BOOL FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer FlushConsoleInputBuffer.argtypes = [HANDLE] FlushConsoleInputBuffer.restype = BOOL + WaitForSingleObject = _KERNEL32.WaitForSingleObject + WaitForSingleObject.argtypes = [HANDLE, DWORD] + WaitForSingleObject.restype = DWORD + OutHandle = GetStdHandle(STD_OUTPUT_HANDLE) InHandle = GetStdHandle(STD_INPUT_HANDLE) else: @@ -610,9 +721,10 @@ def _win_only(*args, **kwargs): GetStdHandle = _win_only GetConsoleScreenBufferInfo = _win_only ScrollConsoleScreenBuffer = _win_only + GetConsoleMode = _win_only SetConsoleMode = _win_only ReadConsoleInput = _win_only - GetNumberOfConsoleInputEvents = _win_only FlushConsoleInputBuffer = _win_only + WaitForSingleObject = _win_only OutHandle = 0 InHandle = 0 diff --git a/Lib/_pyrepl/windows_eventqueue.py b/Lib/_pyrepl/windows_eventqueue.py new file mode 100644 index 00000000000..d99722f9a16 --- /dev/null +++ b/Lib/_pyrepl/windows_eventqueue.py @@ -0,0 +1,42 @@ +""" +Windows event and VT sequence scanner +""" + +from .base_eventqueue import BaseEventQueue + + +# Reference: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#input-sequences +VT_MAP: dict[bytes, str] = { + b'\x1b[A': 'up', + b'\x1b[B': 'down', + b'\x1b[C': 'right', + b'\x1b[D': 'left', + b'\x1b[1;5D': 'ctrl left', + b'\x1b[1;5C': 'ctrl right', + + b'\x1b[H': 'home', + b'\x1b[F': 'end', + + b'\x7f': 'backspace', + b'\x1b[2~': 'insert', + b'\x1b[3~': 'delete', + b'\x1b[5~': 'page up', + b'\x1b[6~': 'page down', + + b'\x1bOP': 'f1', + b'\x1bOQ': 'f2', + b'\x1bOR': 'f3', + b'\x1bOS': 'f4', + b'\x1b[15~': 'f5', + b'\x1b[17~': 'f6', + b'\x1b[18~': 'f7', + b'\x1b[19~': 'f8', + b'\x1b[20~': 'f9', + b'\x1b[21~': 'f10', + b'\x1b[23~': 'f11', + b'\x1b[24~': 'f12', +} + +class EventQueue(BaseEventQueue): + def __init__(self, encoding: str) -> None: + BaseEventQueue.__init__(self, encoding, VT_MAP) diff --git a/Lib/_strptime.py b/Lib/_strptime.py index a07eb5f9230..fc7e369c3d1 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -472,7 +472,7 @@ def repl(m): if day_of_month_in_format and not year_in_format: import warnings warnings.warn("""\ -Parsing dates involving a day of month without a year specified is ambiguious +Parsing dates involving a day of month without a year specified is ambiguous and fails to parse leap day. The default behavior will change in Python 3.15 to either always raise an exception or to use a different default year (TBD). To avoid trouble, add a specific year to the input & format. @@ -783,18 +783,40 @@ def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"): tt = _strptime(data_string, format)[0] return time.struct_time(tt[:time._STRUCT_TM_ITEMS]) -def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): - """Return a class cls instance based on the input string and the +def _strptime_datetime_date(cls, data_string, format="%a %b %d %Y"): + """Return a date instance based on the input string and the + format string.""" + tt, _, _ = _strptime(data_string, format) + args = tt[:3] + return cls(*args) + +def _parse_tz(tzname, gmtoff, gmtoff_fraction): + tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) + if tzname: + return datetime_timezone(tzdelta, tzname) + else: + return datetime_timezone(tzdelta) + +def _strptime_datetime_time(cls, data_string, format="%H:%M:%S"): + """Return a time instance based on the input string and the format string.""" tt, fraction, gmtoff_fraction = _strptime(data_string, format) tzname, gmtoff = tt[-2:] - args = tt[:6] + (fraction,) - if gmtoff is not None: - tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) - if tzname: - tz = datetime_timezone(tzdelta, tzname) - else: - tz = datetime_timezone(tzdelta) - args += (tz,) + args = tt[3:6] + (fraction,) + if gmtoff is None: + return cls(*args) + else: + tz = _parse_tz(tzname, gmtoff, gmtoff_fraction) + return cls(*args, tz) - return cls(*args) +def _strptime_datetime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): + """Return a datetime instance based on the input string and the + format string.""" + tt, fraction, gmtoff_fraction = _strptime(data_string, format) + tzname, gmtoff = tt[-2:] + args = tt[:6] + (fraction,) + if gmtoff is None: + return cls(*args) + else: + tz = _parse_tz(tzname, gmtoff, gmtoff_fraction) + return cls(*args, tz) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index a5788cdbfae..5c9a0812646 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -47,6 +47,7 @@ class Format(enum.IntEnum): "__cell__", "__owner__", "__stringifier_dict__", + "__resolved_str_cache__", ) @@ -94,6 +95,7 @@ def __init__( # value later. self.__code__ = None self.__ast_node__ = None + self.__resolved_str_cache__ = None def __init_subclass__(cls, /, *args, **kwds): raise TypeError("Cannot subclass ForwardRef") @@ -113,7 +115,7 @@ def evaluate( """ match format: case Format.STRING: - return self.__forward_arg__ + return self.__resolved_str__ case Format.VALUE: is_forwardref_format = False case Format.FORWARDREF: @@ -258,6 +260,24 @@ def __forward_arg__(self): "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" ) + @property + def __resolved_str__(self): + # __forward_arg__ with any names from __extra_names__ replaced + # with the type_repr of the value they represent + if self.__resolved_str_cache__ is None: + resolved_str = self.__forward_arg__ + names = self.__extra_names__ + + if names: + visitor = _ExtraNameFixer(names) + ast_expr = ast.parse(resolved_str, mode="eval").body + node = visitor.visit(ast_expr) + resolved_str = ast.unparse(node) + + self.__resolved_str_cache__ = resolved_str + + return self.__resolved_str_cache__ + @property def __forward_code__(self): if self.__code__ is not None: @@ -279,7 +299,13 @@ def __eq__(self, other): # because dictionaries are not hashable. and self.__globals__ is other.__globals__ and self.__forward_is_class__ == other.__forward_is_class__ - and self.__cell__ == other.__cell__ + # Two separate cells are always considered unequal in forward refs. + and ( + {name: id(cell) for name, cell in self.__cell__.items()} + == {name: id(cell) for name, cell in other.__cell__.items()} + if isinstance(self.__cell__, dict) and isinstance(other.__cell__, dict) + else self.__cell__ is other.__cell__ + ) and self.__owner__ == other.__owner__ and ( (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == @@ -293,7 +319,10 @@ def __hash__(self): self.__forward_module__, id(self.__globals__), # dictionaries are not hashable, so hash by identity self.__forward_is_class__, - tuple(sorted(self.__cell__.items())) if isinstance(self.__cell__, dict) else self.__cell__, + ( # cells are not hashable as well + tuple(sorted([(name, id(cell)) for name, cell in self.__cell__.items()])) + if isinstance(self.__cell__, dict) else id(self.__cell__), + ), self.__owner__, tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, )) @@ -312,7 +341,7 @@ def __repr__(self): extra.append(", is_class=True") if self.__owner__ is not None: extra.append(f", owner={self.__owner__!r}") - return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" + return f"ForwardRef({self.__resolved_str__!r}{''.join(extra)})" _Template = type(t"") @@ -348,6 +377,7 @@ def __init__( self.__cell__ = cell self.__owner__ = owner self.__stringifier_dict__ = stringifier_dict + self.__resolved_str_cache__ = None # Needed for ForwardRef def __convert_to_ast(self, other): if isinstance(other, _Stringifier): @@ -910,7 +940,7 @@ def get_annotations( does not exist, the __annotate__ function is called. The FORWARDREF format uses __annotations__ if it exists and can be evaluated, and otherwise falls back to calling the __annotate__ function. - The SOURCE format tries __annotate__ first, and falls back to + The STRING format tries __annotate__ first, and falls back to using __annotations__, stringified using annotations_to_string(). This function handles several details for you: @@ -1028,13 +1058,26 @@ def get_annotations( obj_globals = obj_locals = unwrap = None if unwrap is not None: + # Use an id-based visited set to detect cycles in the __wrapped__ + # and functools.partial.func chain (e.g. f.__wrapped__ = f). + # On cycle detection we stop and use whatever __globals__ we have + # found so far, mirroring the approach of inspect.unwrap(). + _seen_ids = {id(unwrap)} while True: if hasattr(unwrap, "__wrapped__"): - unwrap = unwrap.__wrapped__ + candidate = unwrap.__wrapped__ + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate continue if functools := sys.modules.get("functools"): if isinstance(unwrap, functools.partial): - unwrap = unwrap.func + candidate = unwrap.func + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate continue break if hasattr(unwrap, "__globals__"): @@ -1096,7 +1139,7 @@ def _rewrite_star_unpack(arg): """If the given argument annotation expression is a star unpack e.g. `'*Ts'` rewrite it to a valid expression. """ - if arg.startswith("*"): + if arg.lstrip().startswith("*"): return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] else: return arg @@ -1141,3 +1184,14 @@ def _get_dunder_annotations(obj): if not isinstance(ann, dict): raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") return ann + + +class _ExtraNameFixer(ast.NodeTransformer): + """Fixer for __extra_names__ items in ForwardRef __repr__ and string evaluation""" + def __init__(self, extra_names): + self.extra_names = extra_names + + def visit_Name(self, node: ast.Name): + if (new_name := self.extra_names.get(node.id, _sentinel)) is not _sentinel: + node = ast.Name(id=type_repr(new_name)) + return node diff --git a/Lib/argparse.py b/Lib/argparse.py index 88c1f5a7ef3..a106a718a99 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -149,6 +149,10 @@ def _copy_items(items): return copy.copy(items) +def _identity(value): + return value + + # =============== # Formatting Help # =============== @@ -200,7 +204,7 @@ def _set_color(self, color): self._decolor = decolor else: self._theme = get_theme(force_no_color=True).argparse - self._decolor = lambda text: text + self._decolor = _identity # =============================== # Section and indentation methods @@ -337,27 +341,17 @@ def _format_usage(self, usage, actions, groups, prefix): elif usage is None: prog = '%(prog)s' % dict(prog=self._prog) - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - + parts, pos_start = self._get_actions_usage_parts(actions, groups) # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) + usage = ' '.join(filter(None, [prog, *parts])) # wrap the usage parts if it's too long text_width = self._width - self._current_indent if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts - opt_parts = self._get_actions_usage_parts(optionals, groups) - pos_parts = self._get_actions_usage_parts(positionals, groups) + opt_parts = parts[:pos_start] + pos_parts = parts[pos_start:] # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -414,110 +408,114 @@ def get_lines(parts, indent, prefix=None): # prefix with 'usage:' return f'{t.usage}{prefix}{t.reset}{usage}\n\n' - def _format_actions_usage(self, actions, groups): - return ' '.join(self._get_actions_usage_parts(actions, groups)) - def _is_long_option(self, string): return len(string) > 2 def _get_actions_usage_parts(self, actions, groups): - # find group indices and identify actions in groups - group_actions = set() - inserts = {} - for group in groups: - if not group._group_actions: - raise ValueError(f'empty group {group}') + """Get usage parts with split index for optionals/positionals. - if all(action.help is SUPPRESS for action in group._group_actions): - continue - - try: - start = min(actions.index(item) for item in group._group_actions) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if set(actions[start:end]) == set(group._group_actions): - group_actions.update(group._group_actions) - inserts[start, end] = group + Returns (parts, pos_start) where pos_start is the index in parts + where positionals begin. + This preserves mutually exclusive group formatting across the + optionals/positionals boundary (gh-75949). + """ + actions = [action for action in actions if action.help is not SUPPRESS] + # group actions by mutually exclusive groups + action_groups = dict.fromkeys(actions) + for group in groups: + for action in group._group_actions: + if action in action_groups: + action_groups[action] = group + # positional arguments keep their position + positionals = [] + for action in actions: + if not action.option_strings: + group = action_groups.pop(action) + if group: + group_actions = [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + [action] + positionals.append((group.required, group_actions)) + else: + positionals.append((None, [action])) + # the remaining optional arguments are sorted by the position of + # the first option in the group + optionals = [] + for action in actions: + if action.option_strings and action in action_groups: + group = action_groups.pop(action) + if group: + group_actions = [action] + [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + optionals.append((group.required, group_actions)) + else: + optionals.append((None, [action])) # collect all actions format strings parts = [] t = self._theme - for action in actions: - - # suppressed arguments are marked with None - if action.help is SUPPRESS: - part = None - - # produce all arg strings - elif not action.option_strings: - default = self._get_default_metavar_for_positional(action) - part = ( - t.summary_action - + self._format_args(action, default) - + t.reset - ) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - if self._is_long_option(option_string): - option_color = t.summary_long_option + pos_start = None + for i, (required, group) in enumerate(optionals + positionals): + start = len(parts) + if i == len(optionals): + pos_start = start + in_group = len(group) > 1 + for action in group: + # produce all arg strings + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + # if it's in a group, strip the outer [] + if in_group: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + part = t.summary_action + part + t.reset + + # produce the first way to invoke the option in brackets else: - option_color = t.summary_short_option + option_string = action.option_strings[0] + if self._is_long_option(option_string): + option_color = t.summary_long_option + else: + option_color = t.summary_short_option - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = action.format_usage() - part = f"{option_color}{part}{t.reset}" + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = action.format_usage() + part = f"{option_color}{part}{t.reset}" - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - part = ( - f"{option_color}{option_string} " - f"{t.summary_label}{args_string}{t.reset}" - ) - - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part - - # add the action string to the list - parts.append(part) - - # group mutually exclusive actions - inserted_separators_indices = set() - for start, end in sorted(inserts, reverse=True): - group = inserts[start, end] - group_parts = [item for item in parts[start:end] if item is not None] - group_size = len(group_parts) - if group.required: - open, close = "()" if group_size > 1 else ("", "") - else: - open, close = "[]" - group_parts[0] = open + group_parts[0] - group_parts[-1] = group_parts[-1] + close - for i, part in enumerate(group_parts[:-1], start=start): - # insert a separator if not already done in a nested group - if i not in inserted_separators_indices: - parts[i] = part + ' |' - inserted_separators_indices.add(i) - parts[start + group_size - 1] = group_parts[-1] - for i in range(start + group_size, end): - parts[i] = None - - # return the usage parts - return [item for item in parts if item is not None] + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = ( + f"{option_color}{option_string} " + f"{t.summary_label}{args_string}{t.reset}" + ) + + # make it look optional if it's not required or in a group + if not (action.required or required or in_group): + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + if in_group: + parts[start] = ('(' if required else '[') + parts[start] + for i in range(start, len(parts) - 1): + parts[i] += ' |' + parts[-1] += ')' if required else ']' + + if pos_start is None: + pos_start = len(parts) + return parts, pos_start def _format_text(self, text): if '%(prog)' in text: @@ -745,11 +743,14 @@ def _get_help_string(self, action): if help is None: help = '' - if '%(default)' not in help: - if action.default is not SUPPRESS: - defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - help += _(' (default: %(default)s)') + if ( + '%(default)' not in help + and action.default is not SUPPRESS + and not action.required + ): + defaulting_nargs = (OPTIONAL, ZERO_OR_MORE) + if action.option_strings or action.nargs in defaulting_nargs: + help += _(' (default: %(default)s)') return help @@ -1552,8 +1553,8 @@ def add_argument(self, *args, **kwargs): f'instance of it must be passed') # raise an error if the metavar does not match the type - if hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._format_args(action, None) except TypeError: @@ -1741,8 +1742,8 @@ def _handle_conflict_resolve(self, action, conflicting_actions): action.container._remove_action(action) def _check_help(self, action): - if action.help and hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if action.help and hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._expand_help(action) except (ValueError, TypeError, KeyError) as exc: @@ -1828,6 +1829,9 @@ def _prog_name(prog=None): if modspec is None: # simple script return _os.path.basename(arg0) + if modspec.name != '__main__' and arg0 != modspec.origin: + # named module executed as main without altering sys.argv[0] + return _os.path.basename(arg0) py = _os.path.basename(_sys.executable) if modspec.name != '__main__': # imported module or package @@ -1897,15 +1901,16 @@ def __init__(self, self.suggest_on_error = suggest_on_error self.color = color + # Cached formatter for validation (avoids repeated _set_color calls) + self._cached_formatter = None + add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) self._optionals = add_group(_('options')) self._subparsers = None # register types - def identity(string): - return string - self.register('type', None, identity) + self.register('type', None, _identity) # add help argument if necessary # (using explicit default to override global argument_default) @@ -2676,7 +2681,7 @@ def _check_value(self, action, value): if value not in choices: args = {'value': str(value), - 'choices': ', '.join(map(str, action.choices))} + 'choices': ', '.join(repr(str(choice)) for choice in action.choices)} msg = _('invalid choice: %(value)r (choose from %(choices)s)') if self.suggest_on_error and isinstance(value, str): @@ -2728,6 +2733,13 @@ def _get_formatter(self): formatter._set_color(self.color) return formatter + def _get_validation_formatter(self): + # Return cached formatter for read-only validation operations + # (_expand_help and _format_args). Avoids repeated slow _set_color calls. + if self._cached_formatter is None: + self._cached_formatter = self._get_formatter() + return self._cached_formatter + # ===================== # Help-printing methods # ===================== diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 03165a425eb..32a5dbae03a 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -10,6 +10,7 @@ from .events import * from .exceptions import * from .futures import * +from .graph import * from .locks import * from .protocols import * from .runners import * @@ -27,6 +28,7 @@ events.__all__ + exceptions.__all__ + futures.__all__ + + graph.__all__ + locks.__all__ + protocols.__all__ + runners.__all__ + @@ -45,3 +47,28 @@ else: from .unix_events import * # pragma: no cover __all__ += unix_events.__all__ + +def __getattr__(name: str): + import warnings + + match name: + case "AbstractEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return events._AbstractEventLoopPolicy + case "DefaultEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + if sys.platform == 'win32': + return windows_events._DefaultEventLoopPolicy + return unix_events._DefaultEventLoopPolicy + case "WindowsSelectorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsSelectorEventLoopPolicy + # Else fall through to the AttributeError below. + case "WindowsProactorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsProactorEventLoopPolicy + # Else fall through to the AttributeError below. + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 8e9af83cc8a..44e14771b11 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,5 +1,7 @@ +import argparse import ast import asyncio +import asyncio.tools import concurrent.futures import contextvars import inspect @@ -10,7 +12,7 @@ import types import warnings -from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] +from _colorize import get_theme from _pyrepl.console import InteractiveColoredConsole from . import futures @@ -84,26 +86,32 @@ def run(self): global return_code try: - banner = ( - f'asyncio REPL {sys.version} on {sys.platform}\n' - f'Use "await" directly instead of "asyncio.run()".\n' - f'Type "help", "copyright", "credits" or "license" ' - f'for more information.\n' - ) + if not sys.flags.quiet: + banner = ( + f'asyncio REPL {sys.version} on {sys.platform}\n' + f'Use "await" directly instead of "asyncio.run()".\n' + f'Type "help", "copyright", "credits" or "license" ' + f'for more information.\n' + ) - console.write(banner) + console.write(banner) - if startup_path := os.getenv("PYTHONSTARTUP"): + if not sys.flags.isolated and (startup_path := os.getenv("PYTHONSTARTUP")): sys.audit("cpython.run_startup", startup_path) - - import tokenize - with tokenize.open(startup_path) as f: - startup_code = compile(f.read(), startup_path, "exec") + try: + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") exec(startup_code, console.locals) + except SystemExit: + raise + except BaseException: + console.showtraceback() ps1 = getattr(sys, "ps1", ">>> ") - if can_colorize() and CAN_USE_PYREPL: - ps1 = f"{ANSIColors.BOLD_MAGENTA}{ps1}{ANSIColors.RESET}" + if CAN_USE_PYREPL: + theme = get_theme().syntax + ps1 = f"{theme.prompt}{ps1}{theme.reset}" console.write(f"{ps1}import asyncio\n") if CAN_USE_PYREPL: @@ -141,6 +149,37 @@ def interrupt(self) -> None: if __name__ == '__main__': + parser = argparse.ArgumentParser( + prog="python3 -m asyncio", + description="Interactive asyncio shell and CLI tools", + color=True, + ) + subparsers = parser.add_subparsers(help="sub-commands", dest="command") + ps = subparsers.add_parser( + "ps", help="Display a table of all pending tasks in a process" + ) + ps.add_argument("pid", type=int, help="Process ID to inspect") + pstree = subparsers.add_parser( + "pstree", help="Display a tree of all pending tasks in a process" + ) + pstree.add_argument("pid", type=int, help="Process ID to inspect") + args = parser.parse_args() + match args.command: + case "ps": + asyncio.tools.display_awaited_by_tasks_table(args.pid) + sys.exit(0) + case "pstree": + asyncio.tools.display_awaited_by_tasks_tree(args.pid) + sys.exit(0) + case None: + pass # continue to the interactive shell + case _: + # shouldn't happen as an invalid command-line wouldn't parse + # but let's keep it for the next person adding a command + print(f"error: unhandled command {args.command}", file=sys.stderr) + parser.print_usage(file=sys.stderr) + sys.exit(1) + sys.audit("cpython.run_stdin") if os.getenv('PYTHON_BASIC_REPL'): @@ -202,4 +241,5 @@ def interrupt(self) -> None: break console.write('exiting asyncio REPL...\n') + loop.close() sys.exit(return_code) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 356fc3d7913..6d1f1f84323 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -458,24 +458,18 @@ def create_future(self): """Create a Future object attached to the loop.""" return futures.Future(loop=self) - def create_task(self, coro, *, name=None, context=None, **kwargs): - """Schedule a coroutine object. + def create_task(self, coro, **kwargs): + """Schedule or begin executing a coroutine object. Return a task object. """ self._check_closed() if self._task_factory is not None: - if context is not None: - kwargs["context"] = context - - task = self._task_factory(self, coro, **kwargs) - task.set_name(name) - - else: - task = tasks.Task(coro, loop=self, name=name, context=context, **kwargs) - if task._source_traceback: - del task._source_traceback[-1] + return self._task_factory(self, coro, **kwargs) + task = tasks.Task(coro, loop=self, **kwargs) + if task._source_traceback: + del task._source_traceback[-1] try: return task finally: @@ -489,10 +483,10 @@ def set_task_factory(self, factory): If factory is None the default task factory will be set. If factory is a callable, it should have a signature matching - '(loop, coro, **kwargs)', where 'loop' will be a reference to the active - event loop, 'coro' will be a coroutine object, and **kwargs will be - arbitrary keyword arguments that should be passed on to Task. - The callable must return a Task. + '(loop, coro, **kwargs)', where 'loop' will be a reference to the + active event loop, 'coro' will be a coroutine object, and **kwargs + will be arbitrary keyword arguments that should be passed on to + Task. The callable must return a Task. """ if factory is not None and not callable(factory): raise TypeError('task factory must be a callable or None') @@ -727,8 +721,8 @@ def run_until_complete(self, future): def stop(self): """Stop running the event loop. - Every callback already scheduled will still run. This simply informs - run_forever to stop looping after a complete iteration. + Every callback already scheduled will still run. This simply + informs run_forever to stop looping after a complete iteration. """ self._stopping = True @@ -841,7 +835,7 @@ def call_soon(self, callback, *args, context=None): def _check_callback(self, callback, method): if (coroutines.iscoroutine(callback) or - coroutines.iscoroutinefunction(callback)): + coroutines._iscoroutinefunction(callback)): raise TypeError( f"coroutines cannot be used with {method}()") if not callable(callback): @@ -878,7 +872,10 @@ def call_soon_threadsafe(self, callback, *args, context=None): self._check_closed() if self._debug: self._check_callback(callback, 'call_soon_threadsafe') - handle = self._call_soon(callback, args, context) + handle = events._ThreadSafeHandle(callback, args, self, context) + self._ready.append(handle) + if handle._source_traceback: + del handle._source_traceback[-1] if handle._source_traceback: del handle._source_traceback[-1] self._write_to_self() @@ -966,7 +963,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): f"and file {file!r} combination") async def _sock_sendfile_fallback(self, sock, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = ( min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE) @@ -1073,12 +1070,12 @@ async def create_connection( Create a streaming transport connection to a given internet host and port: socket family AF_INET or socket.AF_INET6 depending on host (or - family if specified), socket type SOCK_STREAM. protocol_factory must be - a callable returning a protocol instance. + family if specified), socket type SOCK_STREAM. protocol_factory must + be a callable returning a protocol instance. - This method is a coroutine which will try to establish the connection - in the background. When successful, the coroutine returns a - (transport, protocol) pair. + This method is a coroutine which will try to establish the + connection in the background. When successful, the coroutine + returns a (transport, protocol) pair. """ if server_hostname is not None and not ssl: raise ValueError('server_hostname is only meaningful with ssl') @@ -1281,7 +1278,6 @@ async def sendfile(self, transport, file, offset=0, count=None, raise RuntimeError( f"fallback is disabled and native sendfile is not " f"supported for transport {transport!r}") - return await self._sendfile_fallback(transport, file, offset, count) @@ -1290,7 +1286,7 @@ async def _sendfile_native(self, transp, file, offset, count): "sendfile syscall is not supported") async def _sendfile_fallback(self, transp, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = min(count, 16384) if count else 16384 buf = bytearray(blocksize) @@ -1348,6 +1344,17 @@ async def start_tls(self, transport, protocol, sslcontext, *, # have a chance to get called before "ssl_protocol.connection_made()". transport.pause_reading() + # gh-142352: move buffered StreamReader data to SSLProtocol + if server_side: + from .streams import StreamReaderProtocol + if isinstance(protocol, StreamReaderProtocol): + stream_reader = getattr(protocol, '_stream_reader', None) + if stream_reader is not None: + buffer = stream_reader._buffer + if buffer: + ssl_protocol._incoming.write(buffer) + buffer.clear() + transport.set_protocol(ssl_protocol) conmade_cb = self.call_soon(ssl_protocol.connection_made, transport) resume_cb = self.call_soon(transport.resume_reading) @@ -1535,11 +1542,11 @@ async def create_server( The host parameter can be a string, in that case the TCP server is bound to host and port. - The host parameter can also be a sequence of strings and in that case - the TCP server is bound to all hosts of the sequence. If a host - appears multiple times (possibly indirectly e.g. when hostnames - resolve to the same IP address), the server is only bound once to that - host. + The host parameter can also be a sequence of strings and in that + case the TCP server is bound to all hosts of the sequence. If + a host appears multiple times (possibly indirectly e.g. when + hostnames resolve to the same IP address), the server is only bound + once to that host. Return a Server object which can be used to stop the service. @@ -1677,8 +1684,7 @@ async def connect_accepted_socket( raise ValueError( 'ssl_shutdown_timeout is only meaningful with ssl') - if sock is not None: - _check_ssl_socket(sock) + _check_ssl_socket(sock) transport, protocol = await self._create_connection_transport( sock, protocol_factory, ssl, '', server_side=True, diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 321a4e5d5d1..224b1883808 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -265,7 +265,7 @@ def _try_finish(self): # to avoid hanging forever in self._wait as otherwise _exit_waiters # would never be woken up, we wake them up here. for waiter in self._exit_waiters: - if not waiter.cancelled(): + if not waiter.done(): waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): @@ -278,7 +278,7 @@ def _call_connection_lost(self, exc): finally: # wake up futures waiting for wait() for waiter in self._exit_waiters: - if not waiter.cancelled(): + if not waiter.done(): waiter.set_result(self._returncode) self._exit_waiters = None self._loop = None diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index ab4f30eb51b..6727065bbe3 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -19,6 +19,15 @@ def _is_debug_mode(): def iscoroutinefunction(func): """Return True if func is a decorated coroutine function.""" + import warnings + warnings._deprecated("asyncio.iscoroutinefunction", + f"{warnings._DEPRECATED_MSG}; " + "use inspect.iscoroutinefunction() instead", + remove=(3,16)) + return _iscoroutinefunction(func) + + +def _iscoroutinefunction(func): return (inspect.iscoroutinefunction(func) or getattr(func, '_is_coroutine', None) is _is_coroutine) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 0d1591e4604..42d8408a38d 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -5,14 +5,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io __all__ = ( - 'AbstractEventLoopPolicy', - 'AbstractEventLoop', 'AbstractServer', - 'Handle', 'TimerHandle', - 'get_event_loop_policy', 'set_event_loop_policy', - 'get_event_loop', 'set_event_loop', 'new_event_loop', - 'get_child_watcher', 'set_child_watcher', - '_set_running_loop', 'get_running_loop', - '_get_running_loop', + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "get_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "set_event_loop", + "new_event_loop", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", ) import contextvars @@ -22,6 +26,7 @@ import subprocess import sys import threading +import warnings from . import format_helpers @@ -104,6 +109,34 @@ def _run(self): self._loop.call_exception_handler(context) self = None # Needed to break cycles when an exception occurs. +# _ThreadSafeHandle is used for callbacks scheduled with call_soon_threadsafe +# and is thread safe unlike Handle which is not thread safe. +class _ThreadSafeHandle(Handle): + + __slots__ = ('_lock',) + + def __init__(self, callback, args, loop, context=None): + super().__init__(callback, args, loop, context) + self._lock = threading.RLock() + + def cancel(self): + with self._lock: + return super().cancel() + + def cancelled(self): + with self._lock: + return super().cancelled() + + def _run(self): + # The event loop checks for cancellation without holding the lock + # It is possible that the handle is cancelled after the check + # but before the callback is called so check it again after acquiring + # the lock and return without calling the callback if it is cancelled. + with self._lock: + if self._cancelled: + return + return super()._run() + class TimerHandle(Handle): """Object returned by timed callback registration methods.""" @@ -341,8 +374,8 @@ async def create_server( If host is an empty string or None all interfaces are assumed and a list of multiple sockets will be returned (most likely - one for IPv4 and another one for IPv6). The host parameter can also be - a sequence (e.g. list) of hosts to bind to. + one for IPv4 and another one for IPv6). The host parameter can also + be a sequence (e.g. list) of hosts to bind to. family can be set to either AF_INET or AF_INET6 to force the socket to use IPv4 or IPv6. If not set it will be determined @@ -382,8 +415,9 @@ async def create_server( start_serving set to True (default) causes the created server to start accepting connections immediately. When set to False, - the user should await Server.start_serving() or Server.serve_forever() - to make the server to start accepting connections. + the user should await Server.start_serving() or + Server.serve_forever() to make the server to start accepting + connections. """ raise NotImplementedError @@ -446,8 +480,9 @@ async def create_unix_server( start_serving set to True (default) causes the created server to start accepting connections immediately. When set to False, - the user should await Server.start_serving() or Server.serve_forever() - to make the server to start accepting connections. + the user should await Server.start_serving() or + Server.serve_forever() to make the server to start accepting + connections. """ raise NotImplementedError @@ -478,8 +513,8 @@ async def create_datagram_endpoint(self, protocol_factory, protocol_factory must be a callable returning a protocol instance. - socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending on - host (or family if specified), socket type SOCK_DGRAM. + socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending + on host (or family if specified), socket type SOCK_DGRAM. reuse_address tells the kernel to reuse a local socket in TIME_WAIT state, without waiting for its natural timeout to @@ -519,7 +554,8 @@ async def connect_read_pipe(self, protocol_factory, pipe): async def connect_write_pipe(self, protocol_factory, pipe): """Register write pipe in event loop. - protocol_factory should instantiate object with BaseProtocol interface. + protocol_factory should instantiate object with BaseProtocol + interface. Pipe is file-like object already switched to nonblocking. Return pair (transport, protocol), where transport support WriteTransport interface.""" @@ -629,7 +665,7 @@ def set_debug(self, enabled): raise NotImplementedError -class AbstractEventLoopPolicy: +class _AbstractEventLoopPolicy: """Abstract policy for accessing the event loop.""" def get_event_loop(self): @@ -652,18 +688,7 @@ def new_event_loop(self): the current context, set_event_loop must be called explicitly.""" raise NotImplementedError - # Child processes handling (Unix only). - - def get_child_watcher(self): - "Get the watcher for child processes." - raise NotImplementedError - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - raise NotImplementedError - - -class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): +class _BaseDefaultEventLoopPolicy(_AbstractEventLoopPolicy): """Default policy implementation for accessing the event loop. In this policy, each thread has its own event loop. However, we @@ -680,7 +705,6 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): class _Local(threading.local): _loop = None - _set_called = False def __init__(self): self._local = self._Local() @@ -690,28 +714,6 @@ def get_event_loop(self): Returns an instance of EventLoop or raises an exception. """ - if (self._local._loop is None and - not self._local._set_called and - threading.current_thread() is threading.main_thread()): - stacklevel = 2 - try: - f = sys._getframe(1) - except AttributeError: - pass - else: - # Move up the call stack so that the warning is attached - # to the line outside asyncio itself. - while f: - module = f.f_globals.get('__name__') - if not (module == 'asyncio' or module.startswith('asyncio.')): - break - f = f.f_back - stacklevel += 1 - import warnings - warnings.warn('There is no current event loop', - DeprecationWarning, stacklevel=stacklevel) - self.set_event_loop(self.new_event_loop()) - if self._local._loop is None: raise RuntimeError('There is no current event loop in thread %r.' % threading.current_thread().name) @@ -720,7 +722,6 @@ def get_event_loop(self): def set_event_loop(self, loop): """Set the event loop.""" - self._local._set_called = True if loop is not None and not isinstance(loop, AbstractEventLoop): raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'") self._local._loop = loop @@ -790,34 +791,36 @@ def _init_event_loop_policy(): global _event_loop_policy with _lock: if _event_loop_policy is None: # pragma: no branch - from . import DefaultEventLoopPolicy - _event_loop_policy = DefaultEventLoopPolicy() + if sys.platform == 'win32': + from .windows_events import _DefaultEventLoopPolicy + else: + from .unix_events import _DefaultEventLoopPolicy + _event_loop_policy = _DefaultEventLoopPolicy() -def get_event_loop_policy(): +def _get_event_loop_policy(): """Get the current event loop policy.""" if _event_loop_policy is None: _init_event_loop_policy() return _event_loop_policy +def get_event_loop_policy(): + warnings._deprecated('asyncio.get_event_loop_policy', remove=(3, 16)) + return _get_event_loop_policy() def _set_event_loop_policy(policy): """Set the current event loop policy. If policy is None, the default policy is restored.""" global _event_loop_policy - if policy is not None and not isinstance(policy, AbstractEventLoopPolicy): + if policy is not None and not isinstance(policy, _AbstractEventLoopPolicy): raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'") _event_loop_policy = policy - def set_event_loop_policy(policy): - """Set the current event loop policy. - - If policy is None, the default policy is restored.""" + warnings._deprecated('asyncio.set_event_loop_policy', remove=(3,16)) _set_event_loop_policy(policy) - def get_event_loop(): """Return an asyncio event loop. @@ -831,28 +834,17 @@ def get_event_loop(): current_loop = _get_running_loop() if current_loop is not None: return current_loop - return get_event_loop_policy().get_event_loop() + return _get_event_loop_policy().get_event_loop() def set_event_loop(loop): """Equivalent to calling get_event_loop_policy().set_event_loop(loop).""" - get_event_loop_policy().set_event_loop(loop) + _get_event_loop_policy().set_event_loop(loop) def new_event_loop(): """Equivalent to calling get_event_loop_policy().new_event_loop().""" - return get_event_loop_policy().new_event_loop() - - -def get_child_watcher(): - """Equivalent to calling get_event_loop_policy().get_child_watcher().""" - return get_event_loop_policy().get_child_watcher() - - -def set_child_watcher(watcher): - """Equivalent to calling - get_event_loop_policy().set_child_watcher(watcher).""" - return get_event_loop_policy().set_child_watcher(watcher) + return _get_event_loop_policy().new_event_loop() # Alias pure-Python implementations for testing purposes. @@ -882,7 +874,7 @@ def set_child_watcher(watcher): def on_fork(): # Reset the loop and wakeupfd in the forked child process. if _event_loop_policy is not None: - _event_loop_policy._local = BaseDefaultEventLoopPolicy._Local() + _event_loop_policy._local = _BaseDefaultEventLoopPolicy._Local() _set_running_loop(None) signal.set_wakeup_fd(-1) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 51932639097..e8a99556b18 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -2,6 +2,7 @@ __all__ = ( 'Future', 'wrap_future', 'isfuture', + 'future_add_to_awaited_by', 'future_discard_from_awaited_by', ) import concurrent.futures @@ -43,7 +44,6 @@ class Future: - This class is not compatible with the wait() and as_completed() methods in the concurrent.futures package. - (In Python 3.4 or later we may be able to unify the implementations.) """ # Class variables serving as defaults for instance variables. @@ -61,12 +61,15 @@ class Future: # the Future protocol (i.e. is intended to be duck-type compatible). # The value must also be not-None, to enable a subclass to declare # that it is not compatible by setting this to None. - # - It is set by __iter__() below so that Task._step() can tell + # - It is set by __iter__() below so that Task.__step() can tell # the difference between - # `await Future()` or`yield from Future()` (correct) vs. + # `await Future()` or `yield from Future()` (correct) vs. # `yield Future()` (incorrect). _asyncio_future_blocking = False + # Used by the capture_call_stack() API. + __asyncio_awaited_by = None + __log_traceback = False def __init__(self, *, loop=None): @@ -116,6 +119,12 @@ def _log_traceback(self, val): raise ValueError('_log_traceback can only be set to False') self.__log_traceback = False + @property + def _asyncio_awaited_by(self): + if self.__asyncio_awaited_by is None: + return None + return frozenset(self.__asyncio_awaited_by) + def get_loop(self): """Return the event loop the Future is bound to.""" loop = self._loop @@ -383,7 +392,7 @@ def _set_state(future, other): def _call_check_cancel(destination): if destination.cancelled(): - if source_loop is None or source_loop is dest_loop: + if source_loop is None or source_loop is events._get_running_loop(): source.cancel() else: source_loop.call_soon_threadsafe(source.cancel) @@ -392,7 +401,7 @@ def _call_set_state(source): if (destination.cancelled() and dest_loop is not None and dest_loop.is_closed()): return - if dest_loop is None or dest_loop is source_loop: + if dest_loop is None or dest_loop is events._get_running_loop(): _set_state(destination, source) else: if dest_loop.is_closed(): @@ -416,6 +425,49 @@ def wrap_future(future, *, loop=None): return new_future +def future_add_to_awaited_by(fut, waiter, /): + """Record that `fut` is awaited on by `waiter`.""" + # For the sake of keeping the implementation minimal and assuming + # that most of asyncio users use the built-in Futures and Tasks + # (or their subclasses), we only support native Future objects + # and their subclasses. + # + # Longer version: tracking requires storing the caller-callee + # dependency somewhere. One obvious choice is to store that + # information right in the future itself in a dedicated attribute. + # This means that we'd have to require all duck-type compatible + # futures to implement a specific attribute used by asyncio for + # the book keeping. Another solution would be to store that in + # a global dictionary. The downside here is that that would create + # strong references and any scenario where the "add" call isn't + # followed by a "discard" call would lead to a memory leak. + # Using WeakDict would resolve that issue, but would complicate + # the C code (_asynciomodule.c). The bottom line here is that + # it's not clear that all this work would be worth the effort. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._Future__asyncio_awaited_by is None: + fut._Future__asyncio_awaited_by = set() + fut._Future__asyncio_awaited_by.add(waiter) + + +def future_discard_from_awaited_by(fut, waiter, /): + """Record that `fut` is no longer awaited on by `waiter`.""" + # See the comment in "future_add_to_awaited_by()" body for + # details on implementation. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._Future__asyncio_awaited_by is not None: + fut._Future__asyncio_awaited_by.discard(waiter) + + +_py_future_add_to_awaited_by = future_add_to_awaited_by +_py_future_discard_from_awaited_by = future_discard_from_awaited_by + try: import _asyncio except ImportError: @@ -423,3 +475,7 @@ def wrap_future(future, *, loop=None): else: # _CFuture is needed for tests. Future = _CFuture = _asyncio.Future + future_add_to_awaited_by = _asyncio.future_add_to_awaited_by + future_discard_from_awaited_by = _asyncio.future_discard_from_awaited_by + _c_future_add_to_awaited_by = future_add_to_awaited_by + _c_future_discard_from_awaited_by = future_discard_from_awaited_by diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py new file mode 100644 index 00000000000..35f7fa62140 --- /dev/null +++ b/Lib/asyncio/graph.py @@ -0,0 +1,276 @@ +"""Introspection utils for tasks call graphs.""" + +import dataclasses +import io +import sys +import types + +from . import events +from . import futures +from . import tasks + +__all__ = ( + 'capture_call_graph', + 'format_call_graph', + 'print_call_graph', + 'FrameCallGraphEntry', + 'FutureCallGraph', +) + +# Sadly, we can't re-use the traceback module's datastructures as those +# are tailored for error reporting, whereas we need to represent an +# async call graph. +# +# Going with pretty verbose names as we'd like to export them to the +# top level asyncio namespace, and want to avoid future name clashes. + + +@dataclasses.dataclass(frozen=True, slots=True) +class FrameCallGraphEntry: + frame: types.FrameType + + +@dataclasses.dataclass(frozen=True, slots=True) +class FutureCallGraph: + future: futures.Future + call_stack: tuple["FrameCallGraphEntry", ...] + awaited_by: tuple["FutureCallGraph", ...] + + +def _build_graph_for_future( + future: futures.Future, + *, + limit: int | None = None, +) -> FutureCallGraph: + if not isinstance(future, futures.Future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + + coro = None + if get_coro := getattr(future, 'get_coro', None): + coro = get_coro() if limit != 0 else None + + st: list[FrameCallGraphEntry] = [] + awaited_by: list[FutureCallGraph] = [] + + while coro is not None: + if hasattr(coro, 'cr_await'): + # A native coroutine or duck-type compatible iterator + st.append(FrameCallGraphEntry(coro.cr_frame)) + coro = coro.cr_await + elif hasattr(coro, 'ag_await'): + # A native async generator or duck-type compatible iterator + st.append(FrameCallGraphEntry(coro.cr_frame)) + coro = coro.ag_await + else: + break + + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + + if limit is not None: + if limit > 0: + st = st[:limit] + elif limit < 0: + st = st[limit:] + st.reverse() + return FutureCallGraph(future, tuple(st), tuple(awaited_by)) + + +def capture_call_graph( + future: futures.Future | None = None, + /, + *, + depth: int = 1, + limit: int | None = None, +) -> FutureCallGraph | None: + """Capture the async call graph for the current task or the provided Future. + + The graph is represented with three data structures: + + * FutureCallGraph(future, call_stack, awaited_by) + + Where 'future' is an instance of asyncio.Future or asyncio.Task. + + 'call_stack' is a tuple of FrameGraphEntry objects. + + 'awaited_by' is a tuple of FutureCallGraph objects. + + * FrameCallGraphEntry(frame) + + Where 'frame' is a frame object of a regular Python function + in the call stack. + + Receives an optional 'future' argument. If not passed, + the current task will be used. If there's no current task, the function + returns None. + + If "capture_call_graph()" is introspecting *the current task*, the + optional keyword-only 'depth' argument can be used to skip the specified + number of frames from top of the stack. + + If the optional keyword-only 'limit' argument is provided, each call + stack in the resulting graph is truncated to include at most + ``abs(limit)`` entries. If 'limit' is positive, the entries left are + the closest to the invocation point. If 'limit' is negative, the + topmost entries are left. If 'limit' is omitted or None, all entries + are present. If 'limit' is 0, the call stack is not captured at all, + only "awaited by" information is present. + """ + + loop = events._get_running_loop() + + if future is not None: + # Check if we're in a context of a running event loop; + # if yes - check if the passed future is the currently + # running task or not. + if loop is None or future is not tasks.current_task(loop=loop): + return _build_graph_for_future(future, limit=limit) + # else: future is the current task, move on. + else: + if loop is None: + raise RuntimeError( + 'capture_call_graph() is called outside of a running ' + 'event loop and no *future* to introspect was provided') + future = tasks.current_task(loop=loop) + + if future is None: + # This isn't a generic call stack introspection utility. If we + # can't determine the current task and none was provided, we + # just return. + return None + + if not isinstance(future, futures.Future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + + call_stack: list[FrameCallGraphEntry] = [] + + f = sys._getframe(depth) if limit != 0 else None + try: + while f is not None: + is_async = f.f_generator is not None + call_stack.append(FrameCallGraphEntry(f)) + + if is_async: + if f.f_back is not None and f.f_back.f_generator is None: + # We've reached the bottom of the coroutine stack, which + # must be the Task that runs it. + break + + f = f.f_back + finally: + del f + + awaited_by = [] + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + + if limit is not None: + limit *= -1 + if limit > 0: + call_stack = call_stack[:limit] + elif limit < 0: + call_stack = call_stack[limit:] + + return FutureCallGraph(future, tuple(call_stack), tuple(awaited_by)) + + +def format_call_graph( + future: futures.Future | None = None, + /, + *, + depth: int = 1, + limit: int | None = None, +) -> str: + """Return the async call graph as a string for `future`. + + If `future` is not provided, format the call graph for the current task. + """ + + def render_level(st: FutureCallGraph, buf: list[str], level: int) -> None: + def add_line(line: str) -> None: + buf.append(level * ' ' + line) + + if isinstance(st.future, tasks.Task): + add_line( + f'* Task(name={st.future.get_name()!r}, id={id(st.future):#x})' + ) + else: + add_line( + f'* Future(id={id(st.future):#x})' + ) + + if st.call_stack: + add_line( + f' + Call stack:' + ) + for ste in st.call_stack: + f = ste.frame + + if f.f_generator is None: + f = ste.frame + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {f.f_code.co_qualname}()' + ) + else: + c = f.f_generator + + try: + f = c.cr_frame + code = c.cr_code + tag = 'async' + except AttributeError: + try: + f = c.ag_frame + code = c.ag_code + tag = 'async generator' + except AttributeError: + f = c.gi_frame + code = c.gi_code + tag = 'generator' + + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {tag} {code.co_qualname}()' + ) + + if st.awaited_by: + add_line( + f' + Awaited by:' + ) + for fut in st.awaited_by: + render_level(fut, buf, level + 1) + + graph = capture_call_graph(future, depth=depth + 1, limit=limit) + if graph is None: + return "" + + buf: list[str] = [] + try: + render_level(graph, buf, 0) + finally: + # 'graph' has references to frames so we should + # make sure it's GC'ed as soon as we don't need it. + del graph + return '\n'.join(buf) + +def print_call_graph( + future: futures.Future | None = None, + /, + *, + file: io.Writer[str] | None = None, + depth: int = 1, + limit: int | None = None, +) -> None: + """Print the async call graph for the current task or the provided Future.""" + print(format_call_graph(future, depth=depth, limit=limit), file=file) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 3df4c693a91..1592d30ed45 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -158,10 +158,10 @@ def _wake_up_first(self): class Event(mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Event. - Class implementing event objects. An event manages a flag that can be set - to true with the set() method and reset to false with the clear() method. - The wait() method blocks until the flag is true. The flag is initially - false. + Class implementing event objects. An event manages a flag that can be + set to true with the set() method and reset to false with the clear() + method. The wait() method blocks until the flag is true. The flag is + initially false. """ def __init__(self): @@ -341,9 +341,9 @@ def _notify(self, n): fut.set_result(False) def notify_all(self): - """Wake up all threads waiting on this condition. This method acts - like notify(), but wakes up all waiting threads instead of one. If the - calling thread has not acquired the lock when this method is called, + """Wake up all tasks waiting on this condition. This method acts + like notify(), but wakes up all waiting tasks instead of one. If the + calling task has not acquired the lock when this method is called, a RuntimeError is raised. """ self.notify(len(self._waiters)) @@ -353,9 +353,9 @@ class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin): """A Semaphore implementation. A semaphore manages an internal counter which is decremented by each - acquire() call and incremented by each release() call. The counter - can never go below zero; when acquire() finds that it is zero, it blocks, - waiting until some other thread calls release(). + acquire() call and incremented by each release() call. The counter + can never go below zero; when acquire() finds that it is zero, it + blocks, waiting until some other thread calls release(). Semaphores also support the context management protocol. @@ -511,8 +511,8 @@ async def __aexit__(self, *args): async def wait(self): """Wait for the barrier. - When the specified number of tasks have started waiting, they are all - simultaneously awoken. + When the specified number of tasks have started waiting, they are + all simultaneously awoken. Returns an unique and individual index number from 0 to 'parties-1'. """ async with self._cond: diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index f404273c3ae..6b94975d004 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -756,8 +756,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): offset += blocksize total_sent += blocksize finally: - if total_sent > 0: - file.seek(offset) + file.seek(offset) async def _sendfile_native(self, transp, file, offset, count): resume_reading = transp.is_reading() diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 084fccaaff2..30004f2bc9b 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -33,11 +33,11 @@ class QueueShutDown(Exception): class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. - If maxsize is less than or equal to zero, the queue size is infinite. If it - is an integer greater than 0, then "await put()" will block when the - queue reaches maxsize, until an item is removed by get(). + If maxsize is less than or equal to zero, the queue size is infinite. + If it is an integer greater than 0, then "await put()" will block when + the queue reaches maxsize, until an item is removed by get(). - Unlike the standard library Queue, you can reliably know this Queue's size + Unlike queue.Queue, you can reliably know this Queue's size with qsize(), since your single-threaded asyncio application won't be interrupted between calling qsize() and doing an operation on the Queue. """ @@ -174,8 +174,8 @@ async def get(self): If queue is empty, wait until an item is available. - Raises QueueShutDown if the queue has been shut down and is empty, or - if the queue has been shut down immediately. + Raises QueueShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. """ while self.empty(): if self._is_shutdown and self.empty(): @@ -203,10 +203,11 @@ async def get(self): def get_nowait(self): """Remove and return an item from the queue. - Return an item if one is immediately available, else raise QueueEmpty. + Return an item if one is immediately available, else raise + QueueEmpty. - Raises QueueShutDown if the queue has been shut down and is empty, or - if the queue has been shut down immediately. + Raises QueueShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. """ if self.empty(): if self._is_shutdown: @@ -223,12 +224,12 @@ def task_done(self): a subsequent call to task_done() tells the queue that the processing on the task is complete. - If a join() is currently blocking, it will resume when all items have - been processed (meaning that a task_done() call was received for every - item that had been put() into the queue). + If a join() is currently blocking, it will resume when all items + have been processed (meaning that a task_done() call was received + for every item that had been put() into the queue). - Raises ValueError if called more times than there were items placed in - the queue. + Raises ValueError if called more times than there were items placed + in the queue. """ if self._unfinished_tasks <= 0: raise ValueError('task_done() called too many times') @@ -239,10 +240,11 @@ def task_done(self): async def join(self): """Block until all items in the queue have been gotten and processed. - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer calls task_done() to - indicate that the item was retrieved and all work on it is complete. - When the count of unfinished tasks drops to zero, join() unblocks. + The count of unfinished tasks goes up whenever an item is added to + the queue. The count goes down whenever a consumer calls + task_done() to indicate that the item was retrieved and all work on + it is complete. When the count of unfinished tasks drops to zero, + join() unblocks. """ if self._unfinished_tasks > 0: await self._finished.wait() diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 102ae78021b..774a317564d 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -3,6 +3,7 @@ import contextvars import enum import functools +import inspect import threading import signal from . import coroutines @@ -34,7 +35,8 @@ class Runner: with asyncio.Runner(debug=True) as runner: runner.run(main()) - The run() method can be called multiple times within the runner's context. + The run() method can be called multiple times within the runner's + context. This can be useful for interactive console (e.g. IPython), unittest runners, console tools, -- everywhere when async code @@ -84,10 +86,7 @@ def get_loop(self): return self._loop def run(self, coro, *, context=None): - """Run a coroutine inside the embedded event loop.""" - if not coroutines.iscoroutine(coro): - raise ValueError("a coroutine was expected, got {!r}".format(coro)) - + """Run code in the embedded event loop.""" if events._get_running_loop() is not None: # fail fast with short traceback raise RuntimeError( @@ -95,8 +94,19 @@ def run(self, coro, *, context=None): self._lazy_init() + if not coroutines.iscoroutine(coro): + if inspect.isawaitable(coro): + async def _wrap_awaitable(awaitable): + return await awaitable + + coro = _wrap_awaitable(coro) + else: + raise TypeError('An asyncio.Future, a coroutine or an ' + 'awaitable is required') + if context is None: context = self._context + task = self._loop.create_task(coro, context=context) if (threading.current_thread() is threading.main_thread() diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 8701467d413..7c2062e3dd5 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -173,16 +173,20 @@ def _accept_connection( # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. # See https://bugs.python.org/issue27906 for more details. - for _ in range(backlog): + for _ in range(backlog + 1): try: conn, addr = sock.accept() if self._debug: logger.debug("%r got a new connection from %r: %r", server, addr, conn) conn.setblocking(False) - except (BlockingIOError, InterruptedError, ConnectionAbortedError): - # Early exit because the socket accept buffer is empty. - return None + except ConnectionAbortedError: + # Discard connections that were aborted before accept(). + continue + except (BlockingIOError, InterruptedError): + # Early exit because of a signal or + # the socket accept buffer is empty. + return except OSError as exc: # There's nowhere to send the error, so just log it. if exc.errno in (errno.EMFILE, errno.ENFILE, @@ -526,11 +530,12 @@ def _sock_recvfrom_into(self, fut, sock, buf, bufsize): async def sock_sendall(self, sock, data): """Send data to the socket. - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + The socket must be connected to a remote socket. This method + continues to send data from data until either all data has been + sent or an error occurs. None is returned on success. On error, + an exception is raised, and there is no way to determine how much + data, if any, was successfully processed by the receiving end of + the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -579,11 +584,12 @@ def _sock_sendall(self, fut, sock, view, pos): async def sock_sendto(self, sock, data, address): """Send data to the socket. - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + The socket must be connected to a remote socket. This method + continues to send data from data until either all data has been + sent or an error occurs. None is returned on success. On error, + an exception is raised, and there is no way to determine how much + data, if any, was successfully processed by the receiving end of + the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -694,10 +700,11 @@ def _sock_connect_cb(self, fut, sock, address): async def sock_accept(self, sock): """Accept a connection. - The socket must be bound to an address and listening for connections. - The return value is a pair (conn, address) where conn is a new socket - object usable to send and receive data on the connection, and address - is the address bound to the socket on the other end of the connection. + The socket must be bound to an address and listening for + connections. The return value is a pair (conn, address) where + conn is a new socket object usable to send and receive data on the + connection, and address is the address bound to the socket on the + other end of the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 0afed64fdf9..2ad65d8648e 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -8,6 +8,7 @@ from . import exceptions as exceptions_mod from . import locks from . import tasks +from . import futures async def staggered_race(coro_fns, delay, *, loop=None): @@ -63,6 +64,7 @@ async def staggered_race(coro_fns, delay, *, loop=None): """ # TODO: when we have aiter() and anext(), allow async iterables in coro_fns. loop = loop or events.get_running_loop() + parent_task = tasks.current_task(loop) enum_coro_fns = enumerate(coro_fns) winner_result = None winner_index = None @@ -73,6 +75,7 @@ async def staggered_race(coro_fns, delay, *, loop=None): def task_done(task): running_tasks.discard(task) + futures.future_discard_from_awaited_by(task, parent_task) if ( on_completed_fut is not None and not on_completed_fut.done() @@ -110,6 +113,7 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: this_failed = locks.Event() next_ok_to_start = locks.Event() next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed)) + futures.future_add_to_awaited_by(next_task, parent_task) running_tasks.add(next_task) next_task.add_done_callback(task_done) # next_task has been appended to running_tasks so next_task is ok to @@ -148,6 +152,7 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: try: ok_to_start = locks.Event() first_task = loop.create_task(run_one_coro(ok_to_start, None)) + futures.future_add_to_awaited_by(first_task, parent_task) running_tasks.add(first_task) first_task.add_done_callback(task_done) # first_task has been appended to running_tasks so first_task is ok to start. @@ -171,4 +176,4 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: raise propagate_cancellation_error return winner_result, winner_index, exceptions finally: - del exceptions, propagate_cancellation_error, unhandled_exceptions + del exceptions, propagate_cancellation_error, unhandled_exceptions, parent_task diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 64aac4cc50d..dd8f6618623 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -541,17 +541,17 @@ async def _wait_for_data(self, func_name): self._waiter = None async def readline(self): - """Read chunk of data from the stream until newline (b'\n') is found. + r"""Read chunk of data from the stream until newline (b'\n') is found. - On success, return chunk that ends with newline. If only partial + On success, return chunk that ends with newline. If only partial line can be read due to EOF, return incomplete line without - terminating newline. When EOF was reached while no bytes read, empty - bytes object is returned. + terminating newline. When EOF was reached while no bytes read, + empty bytes object is returned. - If limit is reached, ValueError will be raised. In that case, if + If limit is reached, ValueError will be raised. In that case, if newline was found, complete line including newline will be removed - from internal buffer. Else, internal buffer will be cleared. Limit is - compared against part of the line without newline. + from internal buffer. Else, internal buffer will be cleared. + Limit is compared against part of the line without newline. If stream was paused, this function will automatically resume it if needed. diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 8fda6c8d55e..00e8f6d5d1a 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -6,6 +6,7 @@ from . import events from . import exceptions +from . import futures from . import tasks @@ -178,7 +179,7 @@ async def _aexit(self, et, exc): exc = None - def create_task(self, coro, *, name=None, context=None): + def create_task(self, coro, **kwargs): """Create a new task in this group and return it. Similar to `asyncio.create_task`. @@ -192,10 +193,9 @@ def create_task(self, coro, *, name=None, context=None): if self._aborting: coro.close() raise RuntimeError(f"TaskGroup {self!r} is shutting down") - if context is None: - task = self._loop.create_task(coro, name=name) - else: - task = self._loop.create_task(coro, name=name, context=context) + task = self._loop.create_task(coro, **kwargs) + + futures.future_add_to_awaited_by(task, self._parent_task) # Always schedule the done callback even if the task is # already done (e.g. if the coro was able to complete eagerly), @@ -228,6 +228,8 @@ def _abort(self): def _on_task_done(self, task): self._tasks.discard(task) + futures.future_discard_from_awaited_by(task, self._parent_task) + if self._on_completed_fut is not None and not self._tasks: if not self._on_completed_fut.done(): self._on_completed_fut.set_result(True) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index dadcb5b5f36..14d3c1ceb58 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -48,23 +48,8 @@ def all_tasks(loop=None): # capturing the set of eager tasks first, so if an eager task "graduates" # to a regular task in another thread, we don't risk missing it. eager_tasks = list(_eager_tasks) - # Looping over the WeakSet isn't safe as it can be updated from another - # thread, therefore we cast it to list prior to filtering. The list cast - # itself requires iteration, so we repeat it several times ignoring - # RuntimeErrors (which are not very likely to occur). - # See issues 34970 and 36607 for details. - scheduled_tasks = None - i = 0 - while True: - try: - scheduled_tasks = list(_scheduled_tasks) - except RuntimeError: - i += 1 - if i >= 1000: - raise - else: - break - return {t for t in itertools.chain(scheduled_tasks, eager_tasks) + + return {t for t in itertools.chain(_scheduled_tasks, eager_tasks) if futures._get_loop(t) is loop and not t.done()} @@ -125,7 +110,7 @@ def __init__(self, coro, *, loop=None, name=None, context=None, self.__eager_start() else: self._loop.call_soon(self.__step, context=self._context) - _register_task(self) + _py_register_task(self) def __del__(self): if self._state == futures._PENDING and self._log_destroy_pending: @@ -260,39 +245,39 @@ def uncancel(self): return self._num_cancels_requested def __eager_start(self): - prev_task = _swap_current_task(self._loop, self) + prev_task = _py_swap_current_task(self._loop, self) try: - _register_eager_task(self) + _py_register_eager_task(self) try: self._context.run(self.__step_run_and_handle_result, None) finally: - _unregister_eager_task(self) + _py_unregister_eager_task(self) finally: try: - curtask = _swap_current_task(self._loop, prev_task) + curtask = _py_swap_current_task(self._loop, prev_task) assert curtask is self finally: if self.done(): self._coro = None self = None # Needed to break cycles when an exception occurs. else: - _register_task(self) + _py_register_task(self) def __step(self, exc=None): if self.done(): raise exceptions.InvalidStateError( - f'_step(): already done: {self!r}, {exc!r}') + f'__step(): already done: {self!r}, {exc!r}') if self._must_cancel: if not isinstance(exc, exceptions.CancelledError): exc = self._make_cancelled_error() self._must_cancel = False self._fut_waiter = None - _enter_task(self._loop, self) + _py_enter_task(self._loop, self) try: self.__step_run_and_handle_result(exc) finally: - _leave_task(self._loop, self) + _py_leave_task(self._loop, self) self = None # Needed to break cycles when an exception occurs. def __step_run_and_handle_result(self, exc): @@ -337,6 +322,7 @@ def __step_run_and_handle_result(self, exc): self._loop.call_soon( self.__step, new_exc, context=self._context) else: + futures.future_add_to_awaited_by(result, self) result._asyncio_future_blocking = False result.add_done_callback( self.__wakeup, context=self._context) @@ -371,6 +357,7 @@ def __step_run_and_handle_result(self, exc): self = None # Needed to break cycles when an exception occurs. def __wakeup(self, future): + futures.future_discard_from_awaited_by(future, self) try: future.result() except BaseException as exc: @@ -379,7 +366,7 @@ def __wakeup(self, future): else: # Don't pass the value of `future.result()` explicitly, # as `Future.__iter__` and `Future.__await__` don't need it. - # If we call `_step(value, None)` instead of `_step()`, + # If we call `__step(value, None)` instead of `__step()`, # Python eval loop would use `.send(value)` method call, # instead of `__next__()`, which is slower for futures # that return non-generator iterators from their `__iter__`. @@ -399,19 +386,13 @@ def __wakeup(self, future): Task = _CTask = _asyncio.Task -def create_task(coro, *, name=None, context=None): +def create_task(coro, **kwargs): """Schedule the execution of a coroutine object in a spawn task. Return a Task object. """ loop = events.get_running_loop() - if context is None: - # Use legacy API if context is not needed - task = loop.create_task(coro, name=name) - else: - task = loop.create_task(coro, name=name, context=context) - - return task + return loop.create_task(coro, **kwargs) # wait() and as_completed() similar to those in PEP 3148. @@ -517,6 +498,7 @@ async def _wait(fs, timeout, return_when, loop): if timeout is not None: timeout_handle = loop.call_later(timeout, _release_waiter, waiter) counter = len(fs) + cur_task = current_task() def _on_completion(f): nonlocal counter @@ -529,9 +511,11 @@ def _on_completion(f): timeout_handle.cancel() if not waiter.done(): waiter.set_result(None) + futures.future_discard_from_awaited_by(f, cur_task) for f in fs: f.add_done_callback(_on_completion) + futures.future_add_to_awaited_by(f, cur_task) try: await waiter @@ -640,10 +624,11 @@ def as_completed(fs, *, timeout=None): Run the supplied awaitables concurrently. The returned object can be iterated to obtain the results of the awaitables as they finish. - The object returned can be iterated as an asynchronous iterator or a plain - iterator. When asynchronous iteration is used, the originally-supplied - awaitables are yielded if they are tasks or futures. This makes it easy to - correlate previously-scheduled tasks with their results: + The object returned can be iterated as an asynchronous iterator or + a plain iterator. When asynchronous iteration is used, the + originally-supplied awaitables are yielded if they are tasks or + futures. This makes it easy to correlate previously-scheduled tasks + with their results: ipv4_connect = create_task(open_connection("127.0.0.1", 80)) ipv6_connect = create_task(open_connection("::1", 80)) @@ -659,26 +644,27 @@ def as_completed(fs, *, timeout=None): else: print("IPv4 connection established.") - During asynchronous iteration, implicitly-created tasks will be yielded for - supplied awaitables that aren't tasks or futures. + During asynchronous iteration, implicitly-created tasks will be + yielded for supplied awaitables that aren't tasks or futures. - When used as a plain iterator, each iteration yields a new coroutine that - returns the result or raises the exception of the next completed awaitable. - This pattern is compatible with Python versions older than 3.13: + When used as a plain iterator, each iteration yields a new coroutine + that returns the result or raises the exception of the next completed + awaitable. This pattern is compatible with Python versions older than + 3.13: ipv4_connect = create_task(open_connection("127.0.0.1", 80)) ipv6_connect = create_task(open_connection("::1", 80)) tasks = [ipv4_connect, ipv6_connect] for next_connect in as_completed(tasks): - # next_connect is not one of the original task objects. It must be - # awaited to obtain the result value or raise the exception of the - # awaitable that finishes next. + # next_connect is not one of the original task objects. It must + # be awaited to obtain the result value or raise the exception + # of the awaitable that finishes next. reader, writer = await next_connect - A TimeoutError is raised if the timeout occurs before all awaitables are - done. This is raised by the async for loop during asynchronous iteration or - by the coroutines yielded during plain iteration. + A TimeoutError is raised if the timeout occurs before all awaitables + are done. This is raised by the async for loop during asynchronous + iteration or by the coroutines yielded during plain iteration. """ if inspect.isawaitable(fs): raise TypeError( @@ -817,10 +803,19 @@ def gather(*coros_or_futures, return_exceptions=False): outer.set_result([]) return outer - def _done_callback(fut): + loop = events._get_running_loop() + if loop is not None: + cur_task = current_task(loop) + else: + cur_task = None + + def _done_callback(fut, cur_task=cur_task): nonlocal nfinished nfinished += 1 + if cur_task is not None: + futures.future_discard_from_awaited_by(fut, cur_task) + if outer is None or outer.done(): if not fut.cancelled(): # Mark exception retrieved. @@ -877,7 +872,6 @@ def _done_callback(fut): nfuts = 0 nfinished = 0 done_futs = [] - loop = None outer = None # bpo-46672 for arg in coros_or_futures: if arg not in arg_to_fut: @@ -890,12 +884,13 @@ def _done_callback(fut): # can't control it, disable the "destroy pending task" # warning. fut._log_destroy_pending = False - nfuts += 1 arg_to_fut[arg] = fut if fut.done(): done_futs.append(fut) else: + if cur_task is not None: + futures.future_add_to_awaited_by(fut, cur_task) fut.add_done_callback(_done_callback) else: @@ -915,6 +910,25 @@ def _done_callback(fut): return outer +def _log_on_exception(fut): + if fut.cancelled(): + return + + exc = fut.exception() + if exc is None: + return + + context = { + 'message': + f'{exc.__class__.__name__} exception in shielded future', + 'exception': exc, + 'future': fut, + } + if fut._source_traceback: + context['source_traceback'] = fut._source_traceback + fut._loop.call_exception_handler(context) + + def shield(arg): """Wait for a future, shielding it from cancellation. @@ -955,11 +969,16 @@ def shield(arg): loop = futures._get_loop(inner) outer = loop.create_future() + if loop is not None and (cur_task := current_task(loop)) is not None: + futures.future_add_to_awaited_by(inner, cur_task) + else: + cur_task = None + + def _clear_awaited_by_callback(inner): + futures.future_discard_from_awaited_by(inner, cur_task) + def _inner_done_callback(inner): if outer.cancelled(): - if not inner.cancelled(): - # Mark inner's result as retrieved. - inner.exception() return if inner.cancelled(): @@ -971,10 +990,16 @@ def _inner_done_callback(inner): else: outer.set_result(inner.result()) - def _outer_done_callback(outer): if not inner.done(): inner.remove_done_callback(_inner_done_callback) + # Keep only one callback to log on cancel + inner.remove_done_callback(_log_on_exception) + inner.add_done_callback(_log_on_exception) + + if cur_task is not None: + inner.add_done_callback(_clear_awaited_by_callback) + inner.add_done_callback(_inner_done_callback) outer.add_done_callback(_outer_done_callback) @@ -1007,25 +1032,26 @@ def callback(): def create_eager_task_factory(custom_task_constructor): """Create a function suitable for use as a task factory on an event-loop. - Example usage: + Example usage: - loop.set_task_factory( - asyncio.create_eager_task_factory(my_task_constructor)) + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) - Now, tasks created will be started immediately (rather than being first - scheduled to an event loop). The constructor argument can be any callable - that returns a Task-compatible object and has a signature compatible - with `Task.__init__`; it must have the `eager_start` keyword argument. + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any + callable that returns a Task-compatible object and has a signature + compatible with `Task.__init__`; it must have the `eager_start` + keyword argument. - Most applications will use `Task` for `custom_task_constructor` and in - this case there's no need to call `create_eager_task_factory()` - directly. Instead the global `eager_task_factory` instance can be - used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. - """ + Most applications will use `Task` for `custom_task_constructor` and in + this case there's no need to call `create_eager_task_factory()` + directly. Instead the global `eager_task_factory` instance can be + used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. + """ - def factory(loop, coro, *, name=None, context=None): + def factory(loop, coro, *, eager_start=True, **kwargs): return custom_task_constructor( - coro, loop=loop, name=name, context=context, eager_start=True) + coro, loop=loop, eager_start=eager_start, **kwargs) return factory @@ -1097,14 +1123,13 @@ def _unregister_eager_task(task): _py_enter_task = _enter_task _py_leave_task = _leave_task _py_swap_current_task = _swap_current_task - +_py_all_tasks = all_tasks try: from _asyncio import (_register_task, _register_eager_task, _unregister_task, _unregister_eager_task, _enter_task, _leave_task, _swap_current_task, - _scheduled_tasks, _eager_tasks, _current_tasks, - current_task) + current_task, all_tasks) except ImportError: pass else: @@ -1116,3 +1141,4 @@ def _unregister_eager_task(task): _c_enter_task = _enter_task _c_leave_task = _leave_task _c_swap_current_task = _swap_current_task + _c_all_tasks = all_tasks diff --git a/Lib/asyncio/threads.py b/Lib/asyncio/threads.py index db048a8231d..5001351b097 100644 --- a/Lib/asyncio/threads.py +++ b/Lib/asyncio/threads.py @@ -17,7 +17,8 @@ async def to_thread(func, /, *args, **kwargs): allowing context variables from the main thread to be accessed in the separate thread. - Return a coroutine that can be awaited to get the eventual result of *func*. + Return a coroutine that can be awaited to get the eventual result of + *func*. """ loop = events.get_running_loop() ctx = contextvars.copy_context() diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index e6f5100691d..65ddc285abd 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -1,7 +1,6 @@ import enum from types import TracebackType -from typing import final, Optional, Type from . import events from . import exceptions @@ -23,14 +22,14 @@ class _State(enum.Enum): EXITED = "finished" -@final class Timeout: """Asynchronous context manager for cancelling overdue coroutines. - Use `timeout()` or `timeout_at()` rather than instantiating this class directly. + Use `timeout()` or `timeout_at()` rather than instantiating this class + directly. """ - def __init__(self, when: Optional[float]) -> None: + def __init__(self, when: float | None) -> None: """Schedule a timeout that will trigger at a given loop time. - If `when` is `None`, the timeout will never trigger. @@ -39,15 +38,15 @@ def __init__(self, when: Optional[float]) -> None: """ self._state = _State.CREATED - self._timeout_handler: Optional[events.TimerHandle] = None - self._task: Optional[tasks.Task] = None + self._timeout_handler: events.TimerHandle | None = None + self._task: tasks.Task | None = None self._when = when - def when(self) -> Optional[float]: + def when(self) -> float | None: """Return the current deadline.""" return self._when - def reschedule(self, when: Optional[float]) -> None: + def reschedule(self, when: float | None) -> None: """Reschedule the timeout.""" if self._state is not _State.ENTERED: if self._state is _State.CREATED: @@ -96,10 +95,10 @@ async def __aenter__(self) -> "Timeout": async def __aexit__( self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType], - ) -> Optional[bool]: + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool | None: assert self._state in (_State.ENTERED, _State.EXPIRING) if self._timeout_handler is not None: @@ -142,7 +141,7 @@ def _insert_timeout_error(exc_val: BaseException) -> None: exc_val = exc_val.__context__ -def timeout(delay: Optional[float]) -> Timeout: +def timeout(delay: float | None) -> Timeout: """Timeout async context manager. Useful in cases when you want to apply timeout logic around block @@ -162,7 +161,7 @@ def timeout(delay: Optional[float]) -> Timeout: return Timeout(loop.time() + delay if delay is not None else None) -def timeout_at(when: Optional[float]) -> Timeout: +def timeout_at(when: float | None) -> Timeout: """Schedule the timeout at absolute time. Like timeout() but argument gives absolute time in the same clock system diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py new file mode 100644 index 00000000000..f39e11fdd51 --- /dev/null +++ b/Lib/asyncio/tools.py @@ -0,0 +1,276 @@ +"""Tools to analyze tasks running in asyncio programs.""" + +from collections import defaultdict, namedtuple +from itertools import count +from enum import Enum +import sys +from _remote_debugging import RemoteUnwinder, FrameInfo + +class NodeType(Enum): + COROUTINE = 1 + TASK = 2 + + +class CycleFoundException(Exception): + """Raised when there is a cycle when drawing the call tree.""" + def __init__( + self, + cycles: list[list[int]], + id2name: dict[int, str], + ) -> None: + super().__init__(cycles, id2name) + self.cycles = cycles + self.id2name = id2name + + + +# ─── indexing helpers ─────────────────────────────────────────── +def _format_stack_entry(elem: str|FrameInfo) -> str: + if not isinstance(elem, str): + if elem.lineno == 0 and elem.filename == "": + return f"{elem.funcname}" + else: + return f"{elem.funcname} {elem.filename}:{elem.lineno}" + return elem + + +def _index(result): + id2name, awaits, task_stacks = {}, [], {} + for awaited_info in result: + for task_info in awaited_info.awaited_by: + task_id = task_info.task_id + task_name = task_info.task_name + id2name[task_id] = task_name + + # Store the internal coroutine stack for this task + if task_info.coroutine_stack: + for coro_info in task_info.coroutine_stack: + call_stack = coro_info.call_stack + internal_stack = [_format_stack_entry(frame) for frame in call_stack] + task_stacks[task_id] = internal_stack + + # Add the awaited_by relationships (external dependencies) + if task_info.awaited_by: + for coro_info in task_info.awaited_by: + call_stack = coro_info.call_stack + parent_task_id = coro_info.task_name + stack = [_format_stack_entry(frame) for frame in call_stack] + awaits.append((parent_task_id, stack, task_id)) + return id2name, awaits, task_stacks + + +def _build_tree(id2name, awaits, task_stacks): + id2label = {(NodeType.TASK, tid): name for tid, name in id2name.items()} + children = defaultdict(list) + cor_nodes = defaultdict(dict) # Maps parent -> {frame_name: node_key} + next_cor_id = count(1) + + def get_or_create_cor_node(parent, frame): + """Get existing coroutine node or create new one under parent""" + if frame in cor_nodes[parent]: + return cor_nodes[parent][frame] + + node_key = (NodeType.COROUTINE, f"c{next(next_cor_id)}") + id2label[node_key] = frame + children[parent].append(node_key) + cor_nodes[parent][frame] = node_key + return node_key + + # Build task dependency tree with coroutine frames + for parent_id, stack, child_id in awaits: + cur = (NodeType.TASK, parent_id) + for frame in reversed(stack): + cur = get_or_create_cor_node(cur, frame) + + child_key = (NodeType.TASK, child_id) + if child_key not in children[cur]: + children[cur].append(child_key) + + # Add coroutine stacks for leaf tasks + awaiting_tasks = {parent_id for parent_id, _, _ in awaits} + for task_id in id2name: + if task_id not in awaiting_tasks and task_id in task_stacks: + cur = (NodeType.TASK, task_id) + for frame in reversed(task_stacks[task_id]): + cur = get_or_create_cor_node(cur, frame) + + return id2label, children + + +def _roots(id2label, children): + all_children = {c for kids in children.values() for c in kids} + return [n for n in id2label if n not in all_children] + +# ─── detect cycles in the task-to-task graph ─────────────────────── +def _task_graph(awaits): + """Return {parent_task_id: {child_task_id, …}, …}.""" + g = defaultdict(set) + for parent_id, _stack, child_id in awaits: + g[parent_id].add(child_id) + return g + + +def _find_cycles(graph): + """ + Depth-first search for back-edges. + + Returns a list of cycles (each cycle is a list of task-ids) or an + empty list if the graph is acyclic. + """ + WHITE, GREY, BLACK = 0, 1, 2 + color = defaultdict(lambda: WHITE) + path, cycles = [], [] + + def dfs(v): + color[v] = GREY + path.append(v) + for w in graph.get(v, ()): + if color[w] == WHITE: + dfs(w) + elif color[w] == GREY: # back-edge → cycle! + i = path.index(w) + cycles.append(path[i:] + [w]) # make a copy + color[v] = BLACK + path.pop() + + for v in list(graph): + if color[v] == WHITE: + dfs(v) + return cycles + + +# ─── PRINT TREE FUNCTION ─────────────────────────────────────── +def get_all_awaited_by(pid): + unwinder = RemoteUnwinder(pid) + return unwinder.get_all_awaited_by() + + +def build_async_tree(result, task_emoji="(T)", cor_emoji=""): + """ + Build a list of strings for pretty-print an async call tree. + + The call tree is produced by `get_all_async_stacks()`, prefixing tasks + with `task_emoji` and coroutine frames with `cor_emoji`. + """ + id2name, awaits, task_stacks = _index(result) + g = _task_graph(awaits) + cycles = _find_cycles(g) + if cycles: + raise CycleFoundException(cycles, id2name) + labels, children = _build_tree(id2name, awaits, task_stacks) + + def pretty(node): + flag = task_emoji if node[0] == NodeType.TASK else cor_emoji + return f"{flag} {labels[node]}" + + def render(node, prefix="", last=True, buf=None): + if buf is None: + buf = [] + buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}") + new_pref = prefix + (" " if last else "│ ") + kids = children.get(node, []) + for i, kid in enumerate(kids): + render(kid, new_pref, i == len(kids) - 1, buf) + return buf + + return [render(root) for root in _roots(labels, children)] + + +def build_task_table(result): + id2name, _, _ = _index(result) + table = [] + + for awaited_info in result: + thread_id = awaited_info.thread_id + for task_info in awaited_info.awaited_by: + # Get task info + task_id = task_info.task_id + task_name = task_info.task_name + + # Build coroutine stack string + frames = [frame for coro in task_info.coroutine_stack + for frame in coro.call_stack] + coro_stack = " -> ".join(_format_stack_entry(x).split(" ")[0] + for x in frames) + + # Handle tasks with no awaiters + if not task_info.awaited_by: + table.append([thread_id, hex(task_id), task_name, coro_stack, + "", "", "0x0"]) + continue + + # Handle tasks with awaiters + for coro_info in task_info.awaited_by: + parent_id = coro_info.task_name + awaiter_frames = [_format_stack_entry(x).split(" ")[0] + for x in coro_info.call_stack] + awaiter_chain = " -> ".join(awaiter_frames) + awaiter_name = id2name.get(parent_id, "Unknown") + parent_id_str = (hex(parent_id) if isinstance(parent_id, int) + else str(parent_id)) + + table.append([thread_id, hex(task_id), task_name, coro_stack, + awaiter_chain, awaiter_name, parent_id_str]) + + return table + +def _print_cycle_exception(exception: CycleFoundException): + print("ERROR: await-graph contains cycles - cannot print a tree!", file=sys.stderr) + print("", file=sys.stderr) + for c in exception.cycles: + inames = " → ".join(exception.id2name.get(tid, hex(tid)) for tid in c) + print(f"cycle: {inames}", file=sys.stderr) + + +def exit_with_permission_help_text(): + """ + Prints a message pointing to platform-specific permission help text and exits the program. + This function is called when a PermissionError is encountered while trying + to attach to a process. + """ + print( + "Error: The specified process cannot be attached to due to insufficient permissions.\n" + "See the Python documentation for details on required privileges and troubleshooting:\n" + "https://docs.python.org/3.14/howto/remote_debugging.html#permission-requirements\n" + ) + sys.exit(1) + + +def _get_awaited_by_tasks(pid: int) -> list: + try: + return get_all_awaited_by(pid) + except RuntimeError as e: + while e.__context__ is not None: + e = e.__context__ + print(f"Error retrieving tasks: {e}") + sys.exit(1) + except PermissionError as e: + exit_with_permission_help_text() + + +def display_awaited_by_tasks_table(pid: int) -> None: + """Build and print a table of all pending tasks under `pid`.""" + + tasks = _get_awaited_by_tasks(pid) + table = build_task_table(tasks) + # Print the table in a simple tabular format + print( + f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine stack':<50} {'awaiter chain':<50} {'awaiter name':<15} {'awaiter id':<15}" + ) + print("-" * 180) + for row in table: + print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") + + +def display_awaited_by_tasks_tree(pid: int) -> None: + """Build and print a tree of all pending tasks under `pid`.""" + + tasks = _get_awaited_by_tasks(pid) + try: + result = build_async_tree(tasks) + except CycleFoundException as e: + _print_cycle_exception(e) + sys.exit(1) + + for tree in result: + print("\n".join(tree)) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 41ccf1b78fb..bfb3da0debb 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -28,10 +28,6 @@ __all__ = ( 'SelectorEventLoop', - 'AbstractChildWatcher', 'SafeChildWatcher', - 'FastChildWatcher', 'PidfdChildWatcher', - 'MultiLoopChildWatcher', 'ThreadedChildWatcher', - 'DefaultEventLoopPolicy', 'EventLoop', ) @@ -58,13 +54,18 @@ def waitstatus_to_exitcode(status): class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): """Unix event loop. - Adds signal handling and UNIX Domain Socket support to SelectorEventLoop. + Adds signal handling and UNIX Domain Socket support to + SelectorEventLoop. """ def __init__(self, selector=None): super().__init__(selector) self._signal_handlers = {} self._unix_server_sockets = {} + if can_use_pidfd(): + self._watcher = _PidfdChildWatcher() + else: + self._watcher = _ThreadedChildWatcher() def close(self): super().close() @@ -94,7 +95,7 @@ def add_signal_handler(self, sig, callback, *args): Raise RuntimeError if there is a problem setting up the handler. """ if (coroutines.iscoroutine(callback) or - coroutines.iscoroutinefunction(callback)): + coroutines._iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) @@ -197,33 +198,22 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None, async def _make_subprocess_transport(self, protocol, args, shell, stdin, stdout, stderr, bufsize, extra=None, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - watcher = events.get_child_watcher() - - with watcher: - if not watcher.is_active(): - # Check early. - # Raising exception before process creation - # prevents subprocess execution if the watcher - # is not ready to handle it. - raise RuntimeError("asyncio.get_child_watcher() is not activated, " - "subprocess support is not installed.") - waiter = self.create_future() - transp = _UnixSubprocessTransport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - waiter=waiter, extra=extra, - **kwargs) - watcher.add_child_handler(transp.get_pid(), - self._child_watcher_callback, transp) - try: - await waiter - except (SystemExit, KeyboardInterrupt): - raise - except BaseException: - transp.close() - await transp._wait() - raise + watcher = self._watcher + waiter = self.create_future() + transp = _UnixSubprocessTransport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + waiter=waiter, extra=extra, + **kwargs) + watcher.add_child_handler(transp.get_pid(), + self._child_watcher_callback, transp) + try: + await waiter + except (SystemExit, KeyboardInterrupt): + raise + except BaseException: + transp.close() + await transp._wait() + raise return transp @@ -395,12 +385,12 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, # order to simplify the common case. self.remove_writer(registered_fd) if fut.cancelled(): - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) return if count: blocksize = count - total_sent if blocksize <= 0: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_result(total_sent) return @@ -434,20 +424,20 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, # plain send(). err = exceptions.SendfileNotAvailableError( "os.sendfile call failed") - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(err) else: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(exc) except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(exc) else: if sent == 0: # EOF - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_result(total_sent) else: offset += sent @@ -458,9 +448,9 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, fd, sock, fileno, offset, count, blocksize, total_sent) - def _sock_sendfile_update_filepos(self, fileno, offset, total_sent): - if total_sent > 0: - os.lseek(fileno, offset, os.SEEK_SET) + def _sock_sendfile_update_filepos(self, fileno, offset): + # After this helper runs, the source fd's lseek pointer is at offset." + os.lseek(fileno, offset, os.SEEK_SET) def _sock_add_cancellation_callback(self, fut, sock): def cb(fut): @@ -865,93 +855,7 @@ def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): stdin_w.close() -class AbstractChildWatcher: - """Abstract base class for monitoring child processes. - - Objects derived from this class monitor a collection of subprocesses and - report their termination or interruption by a signal. - - New callbacks are registered with .add_child_handler(). Starting a new - process must be done within a 'with' block to allow the watcher to suspend - its activity until the new process if fully registered (this is needed to - prevent a race condition in some implementations). - - Example: - with watcher: - proc = subprocess.Popen("sleep 1") - watcher.add_child_handler(proc.pid, callback) - - Notes: - Implementations of this class must be thread-safe. - - Since child watcher objects may catch the SIGCHLD signal and call - waitpid(-1), there should be only one active object per process. - """ - - def __init_subclass__(cls) -> None: - if cls.__module__ != __name__: - warnings._deprecated("AbstractChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def add_child_handler(self, pid, callback, *args): - """Register a new child handler. - - Arrange for callback(pid, returncode, *args) to be called when - process 'pid' terminates. Specifying another callback for the same - process replaces the previous handler. - - Note: callback() must be thread-safe. - """ - raise NotImplementedError() - - def remove_child_handler(self, pid): - """Removes the handler for process 'pid'. - - The function returns True if the handler was successfully removed, - False if there was nothing to remove.""" - - raise NotImplementedError() - - def attach_loop(self, loop): - """Attach the watcher to an event loop. - - If the watcher was previously attached to an event loop, then it is - first detached before attaching to the new loop. - - Note: loop may be None. - """ - raise NotImplementedError() - - def close(self): - """Close the watcher. - - This must be called to make sure that any underlying resource is freed. - """ - raise NotImplementedError() - - def is_active(self): - """Return ``True`` if the watcher is active and is used by the event loop. - - Return True if the watcher is installed and ready to handle process exit - notifications. - - """ - raise NotImplementedError() - - def __enter__(self): - """Enter the watcher's context and allow starting new processes - - This function must return self""" - raise NotImplementedError() - - def __exit__(self, a, b, c): - """Exit the watcher's context""" - raise NotImplementedError() - - -class PidfdChildWatcher(AbstractChildWatcher): +class _PidfdChildWatcher: """Child watcher implementation using Linux's pid file descriptors. This child watcher polls process file descriptors (pidfds) to await child @@ -963,21 +867,6 @@ class PidfdChildWatcher(AbstractChildWatcher): recent (5.3+) kernels. """ - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, exc_traceback): - pass - - def is_active(self): - return True - - def close(self): - pass - - def attach_loop(self, loop): - pass - def add_child_handler(self, pid, callback, *args): loop = events.get_running_loop() pidfd = os.pidfd_open(pid) @@ -1002,386 +891,7 @@ def _do_wait(self, pid, pidfd, callback, args): os.close(pidfd) callback(pid, returncode, *args) - def remove_child_handler(self, pid): - # asyncio never calls remove_child_handler() !!! - # The method is no-op but is implemented because - # abstract base classes require it. - return True - - -class BaseChildWatcher(AbstractChildWatcher): - - def __init__(self): - self._loop = None - self._callbacks = {} - - def close(self): - self.attach_loop(None) - - def is_active(self): - return self._loop is not None and self._loop.is_running() - - def _do_waitpid(self, expected_pid): - raise NotImplementedError() - - def _do_waitpid_all(self): - raise NotImplementedError() - - def attach_loop(self, loop): - assert loop is None or isinstance(loop, events.AbstractEventLoop) - - if self._loop is not None and loop is None and self._callbacks: - warnings.warn( - 'A loop is being detached ' - 'from a child watcher with pending handlers', - RuntimeWarning) - - if self._loop is not None: - self._loop.remove_signal_handler(signal.SIGCHLD) - - self._loop = loop - if loop is not None: - loop.add_signal_handler(signal.SIGCHLD, self._sig_chld) - - # Prevent a race condition in case a child terminated - # during the switch. - self._do_waitpid_all() - - def _sig_chld(self): - try: - self._do_waitpid_all() - except (SystemExit, KeyboardInterrupt): - raise - except BaseException as exc: - # self._loop should always be available here - # as '_sig_chld' is added as a signal handler - # in 'attach_loop' - self._loop.call_exception_handler({ - 'message': 'Unknown exception in SIGCHLD handler', - 'exception': exc, - }) - - -class SafeChildWatcher(BaseChildWatcher): - """'Safe' child watcher implementation. - - This implementation avoids disrupting other code spawning processes by - polling explicitly each process in the SIGCHLD handler instead of calling - os.waitpid(-1). - - This is a safe solution but it has a significant overhead when handling a - big number of children (O(n) each time SIGCHLD is raised) - """ - - def __init__(self): - super().__init__() - warnings._deprecated("SafeChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def close(self): - self._callbacks.clear() - super().close() - - def __enter__(self): - return self - - def __exit__(self, a, b, c): - pass - - def add_child_handler(self, pid, callback, *args): - self._callbacks[pid] = (callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - else: - if pid == 0: - # The child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - - try: - callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - if self._loop.get_debug(): - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - callback(pid, returncode, *args) - - -class FastChildWatcher(BaseChildWatcher): - """'Fast' child watcher implementation. - - This implementation reaps every terminated processes by calling - os.waitpid(-1) directly, possibly breaking other code spawning processes - and waiting for their termination. - - There is no noticeable overhead when handling a big number of children - (O(1) each time a child terminates). - """ - def __init__(self): - super().__init__() - self._lock = threading.Lock() - self._zombies = {} - self._forks = 0 - warnings._deprecated("FastChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def close(self): - self._callbacks.clear() - self._zombies.clear() - super().close() - - def __enter__(self): - with self._lock: - self._forks += 1 - - return self - - def __exit__(self, a, b, c): - with self._lock: - self._forks -= 1 - - if self._forks or not self._zombies: - return - - collateral_victims = str(self._zombies) - self._zombies.clear() - - logger.warning( - "Caught subprocesses termination from unknown pids: %s", - collateral_victims) - - def add_child_handler(self, pid, callback, *args): - assert self._forks, "Must use the context manager" - - with self._lock: - try: - returncode = self._zombies.pop(pid) - except KeyError: - # The child is running. - self._callbacks[pid] = callback, args - return - - # The child is dead already. We can fire the callback. - callback(pid, returncode, *args) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - # Because of signal coalescing, we must keep calling waitpid() as - # long as we're able to reap a child. - while True: - try: - pid, status = os.waitpid(-1, os.WNOHANG) - except ChildProcessError: - # No more child processes exist. - return - else: - if pid == 0: - # A child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - - with self._lock: - try: - callback, args = self._callbacks.pop(pid) - except KeyError: - # unknown child - if self._forks: - # It may not be registered yet. - self._zombies[pid] = returncode - if self._loop.get_debug(): - logger.debug('unknown process %s exited ' - 'with returncode %s', - pid, returncode) - continue - callback = None - else: - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - pid, returncode) - - if callback is None: - logger.warning( - "Caught subprocess termination from unknown pid: " - "%d -> %d", pid, returncode) - else: - callback(pid, returncode, *args) - - -class MultiLoopChildWatcher(AbstractChildWatcher): - """A watcher that doesn't require running loop in the main thread. - - This implementation registers a SIGCHLD signal handler on - instantiation (which may conflict with other code that - install own handler for this signal). - - The solution is safe but it has a significant overhead when - handling a big number of processes (*O(n)* each time a - SIGCHLD is received). - """ - - # Implementation note: - # The class keeps compatibility with AbstractChildWatcher ABC - # To achieve this it has empty attach_loop() method - # and doesn't accept explicit loop argument - # for add_child_handler()/remove_child_handler() - # but retrieves the current loop by get_running_loop() - - def __init__(self): - self._callbacks = {} - self._saved_sighandler = None - warnings._deprecated("MultiLoopChildWatcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - def is_active(self): - return self._saved_sighandler is not None - - def close(self): - self._callbacks.clear() - if self._saved_sighandler is None: - return - - handler = signal.getsignal(signal.SIGCHLD) - if handler != self._sig_chld: - logger.warning("SIGCHLD handler was changed by outside code") - else: - signal.signal(signal.SIGCHLD, self._saved_sighandler) - self._saved_sighandler = None - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def add_child_handler(self, pid, callback, *args): - loop = events.get_running_loop() - self._callbacks[pid] = (loop, callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def attach_loop(self, loop): - # Don't save the loop but initialize itself if called first time - # The reason to do it here is that attach_loop() is called from - # unix policy only for the main thread. - # Main thread is required for subscription on SIGCHLD signal - if self._saved_sighandler is not None: - return - - self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) - if self._saved_sighandler is None: - logger.warning("Previous SIGCHLD handler was set by non-Python code, " - "restore to default handler on watcher close.") - self._saved_sighandler = signal.SIG_DFL - - # Set SA_RESTART to limit EINTR occurrences. - signal.siginterrupt(signal.SIGCHLD, False) - - def _do_waitpid_all(self): - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - debug_log = False - else: - if pid == 0: - # The child process is still alive. - return - - returncode = waitstatus_to_exitcode(status) - debug_log = True - try: - loop, callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - if loop.is_closed(): - logger.warning("Loop %r that handles pid %r is closed", loop, pid) - else: - if debug_log and loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - loop.call_soon_threadsafe(callback, pid, returncode, *args) - - def _sig_chld(self, signum, frame): - try: - self._do_waitpid_all() - except (SystemExit, KeyboardInterrupt): - raise - except BaseException: - logger.warning('Unknown exception in SIGCHLD handler', exc_info=True) - - -class ThreadedChildWatcher(AbstractChildWatcher): +class _ThreadedChildWatcher: """Threaded child watcher implementation. The watcher uses a thread per process @@ -1398,18 +908,6 @@ def __init__(self): self._pid_counter = itertools.count(0) self._threads = {} - def is_active(self): - return True - - def close(self): - pass - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - def __del__(self, _warn=warnings.warn): threads = [thread for thread in list(self._threads.values()) if thread.is_alive()] @@ -1427,15 +925,6 @@ def add_child_handler(self, pid, callback, *args): self._threads[pid] = thread thread.start() - def remove_child_handler(self, pid): - # asyncio never calls remove_child_handler() !!! - # The method is no-op but is implemented because - # abstract base classes require it. - return True - - def attach_loop(self, loop): - pass - def _do_waitpid(self, loop, expected_pid, callback, args): assert expected_pid > 0 @@ -1474,63 +963,11 @@ def can_use_pidfd(): return True -class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): - """UNIX event loop policy with a watcher for child processes.""" +class _UnixDefaultEventLoopPolicy(events._BaseDefaultEventLoopPolicy): + """UNIX event loop policy""" _loop_factory = _UnixSelectorEventLoop - def __init__(self): - super().__init__() - self._watcher = None - - def _init_watcher(self): - with events._lock: - if self._watcher is None: # pragma: no branch - if can_use_pidfd(): - self._watcher = PidfdChildWatcher() - else: - self._watcher = ThreadedChildWatcher() - - def set_event_loop(self, loop): - """Set the event loop. - - As a side effect, if a child watcher was set before, then calling - .set_event_loop() from the main thread will call .attach_loop(loop) on - the child watcher. - """ - - super().set_event_loop(loop) - - if (self._watcher is not None and - threading.current_thread() is threading.main_thread()): - self._watcher.attach_loop(loop) - - def get_child_watcher(self): - """Get the watcher for child processes. - - If not yet set, a ThreadedChildWatcher object is automatically created. - """ - if self._watcher is None: - self._init_watcher() - - warnings._deprecated("get_child_watcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", remove=(3, 14)) - return self._watcher - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - - assert watcher is None or isinstance(watcher, AbstractChildWatcher) - - if self._watcher is not None: - self._watcher.close() - - self._watcher = watcher - warnings._deprecated("set_child_watcher", - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", remove=(3, 14)) - SelectorEventLoop = _UnixSelectorEventLoop -DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy +_DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy EventLoop = SelectorEventLoop diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index bf99bc271c7..0bf7732136f 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -29,8 +29,8 @@ __all__ = ( 'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', - 'DefaultEventLoopPolicy', 'WindowsSelectorEventLoopPolicy', - 'WindowsProactorEventLoopPolicy', 'EventLoop', + '_DefaultEventLoopPolicy', '_WindowsSelectorEventLoopPolicy', + '_WindowsProactorEventLoopPolicy', 'EventLoop', ) @@ -610,6 +610,9 @@ def sendfile(self, sock, file, offset, count): ov = _overlapped.Overlapped(NULL) offset_low = offset & 0xffff_ffff offset_high = (offset >> 32) & 0xffff_ffff + # TransmitFile ignores OVERLAPPED.Offset for handles not opened with + # FILE_FLAG_OVERLAPPED, so seek the CRT file pointer to match. + file.seek(offset) ov.TransmitFile(sock.fileno(), msvcrt.get_osfhandle(file.fileno()), offset_low, offset_high, @@ -891,13 +894,13 @@ def callback(f): SelectorEventLoop = _WindowsSelectorEventLoop -class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsSelectorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = SelectorEventLoop -class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsProactorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = ProactorEventLoop -DefaultEventLoopPolicy = WindowsProactorEventLoopPolicy +_DefaultEventLoopPolicy = _WindowsProactorEventLoopPolicy EventLoop = ProactorEventLoop diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index ef277fac3e2..d6393f0b1ff 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -10,7 +10,6 @@ import msvcrt import os import subprocess -import tempfile import warnings @@ -24,6 +23,7 @@ PIPE = subprocess.PIPE STDOUT = subprocess.STDOUT _mmap_counter = itertools.count() +_MAX_PIPE_ATTEMPTS = 20 # Replacement for os.pipe() using handles instead of fds @@ -31,10 +31,6 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): """Like os.pipe() but with overlapped support and using handles not fds.""" - address = tempfile.mktemp( - prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format( - os.getpid(), next(_mmap_counter))) - if duplex: openmode = _winapi.PIPE_ACCESS_DUPLEX access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE @@ -56,9 +52,20 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): h1 = h2 = None try: - h1 = _winapi.CreateNamedPipe( - address, openmode, _winapi.PIPE_WAIT, - 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + for attempts in itertools.count(): + address = r'\\.\pipe\python-pipe-{:d}-{:d}-{}'.format( + os.getpid(), next(_mmap_counter), os.urandom(8).hex()) + try: + h1 = _winapi.CreateNamedPipe( + address, openmode, _winapi.PIPE_WAIT, + 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + break + except OSError as e: + if attempts >= _MAX_PIPE_ATTEMPTS: + raise + if e.winerror not in (_winapi.ERROR_PIPE_BUSY, + _winapi.ERROR_ACCESS_DENIED): + raise h2 = _winapi.CreateFile( address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, @@ -104,8 +111,9 @@ def fileno(self): def close(self, *, CloseHandle=_winapi.CloseHandle): if self._handle is not None: - CloseHandle(self._handle) + handle = self._handle self._handle = None + CloseHandle(handle) def __del__(self, _warn=warnings.warn): if self._handle is not None: diff --git a/Lib/base64.py b/Lib/base64.py index f95132a4274..0f84420d7a2 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,4 +1,4 @@ -"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" +"""Base16, Base32, Base64 (RFC 4648), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support @@ -147,7 +147,7 @@ def urlsafe_b64decode(s): characters present in the input. ''' _B32_DECODE_MAP01_DOCSTRING = ''' -RFC 3548 allows for optional mapping of the digit 0 (zero) to the +RFC 4648 allows for optional mapping of the digit 0 (zero) to the letter O (oh), and for optional mapping of the digit 1 (one) to either the letter I (eye) or letter L (el). The optional argument map01 when not None, specifies which letter the digit 1 should be @@ -261,7 +261,7 @@ def b32hexdecode(s, casefold=False): extra_args='') -# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns +# RFC 4648, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. def b16encode(s): @@ -325,17 +325,20 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): foldspaces is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Adobe encoding. + feature is not supported by the standard encoding used in PDF. - wrapcol controls whether the output should have newline (b'\\n') characters - added to it. If this is non-zero, each output line will be at most this - many characters long, excluding the trailing newline. + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. - pad controls whether the input is padded to a multiple of 4 before - encoding. Note that the btoa implementation always pads. + pad controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by btoa, + producing an exact multiple of 5 bytes of output. + + adobe controls whether the encoded byte sequence is framed with <~ + and ~>, as in a PostScript base-85 string literal. Note that + while ASCII85Decode streams in PDF documents must be terminated + with ~>, they must not use a leading <~. - adobe controls whether the encoded byte sequence is framed with <~ and ~>, - which is used by the Adobe implementation. """ global _a85chars, _a85chars2 # Delay the initialization of tables to not waste memory @@ -364,12 +367,14 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): """Decode the Ascii85 encoded bytes-like object or ASCII string b. - foldspaces is a flag that specifies whether the 'y' short sequence should be - accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is - not supported by the "standard" Adobe encoding. + foldspaces is a flag that specifies whether the 'y' short sequence + should be accepted as shorthand for 4 consecutive spaces (ASCII + 0x20). This feature is not supported by the standard Ascii85 + encoding used in PDF and PostScript. - adobe controls whether the input sequence is in Adobe Ascii85 format (i.e. - is framed with <~ and ~>). + adobe controls whether the <~ and ~> markers are present. While + the leading <~ is not required, the input must end with ~>, or a + ValueError is raised. ignorechars should be a byte string containing characters to ignore from the input. This should only contain whitespace characters, and by default @@ -442,8 +447,10 @@ def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): def b85encode(b, pad=False): """Encode bytes-like object b in base85 format and return a bytes object. - If pad is true, the input is padded with b'\\0' so its length is a multiple of - 4 bytes before encoding. + The input is padded with b'\0' so its length is a multiple of 4 + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes. """ global _b85chars, _b85chars2 # Delay the initialization of tables to not waste memory diff --git a/Lib/bdb.py b/Lib/bdb.py index f256b56daaa..79da4bab9c9 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -2,7 +2,9 @@ import fnmatch import sys +import threading import os +import weakref from contextlib import contextmanager from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR @@ -15,6 +17,166 @@ class BdbQuit(Exception): """Exception to give up completely.""" +E = sys.monitoring.events + +class _MonitoringTracer: + EVENT_CALLBACK_MAP = { + E.PY_START: 'call', + E.PY_RESUME: 'call', + E.PY_THROW: 'call', + E.LINE: 'line', + E.JUMP: 'jump', + E.PY_RETURN: 'return', + E.PY_YIELD: 'return', + E.PY_UNWIND: 'unwind', + E.RAISE: 'exception', + E.STOP_ITERATION: 'exception', + E.INSTRUCTION: 'opcode', + } + + GLOBAL_EVENTS = E.PY_START | E.PY_RESUME | E.PY_THROW | E.PY_UNWIND | E.RAISE + LOCAL_EVENTS = E.LINE | E.JUMP | E.PY_RETURN | E.PY_YIELD | E.STOP_ITERATION + + def __init__(self): + self._tool_id = sys.monitoring.DEBUGGER_ID + self._name = 'bdbtracer' + self._tracefunc = None + self._disable_current_event = False + self._tracing_thread = None + self._enabled = False + + def start_trace(self, tracefunc): + self._tracefunc = tracefunc + self._tracing_thread = threading.current_thread() + curr_tool = sys.monitoring.get_tool(self._tool_id) + if curr_tool is None: + sys.monitoring.use_tool_id(self._tool_id, self._name) + elif curr_tool == self._name: + sys.monitoring.clear_tool_id(self._tool_id) + else: + raise ValueError('Another debugger is using the monitoring tool') + E = sys.monitoring.events + all_events = 0 + for event, cb_name in self.EVENT_CALLBACK_MAP.items(): + callback = self.callback_wrapper(getattr(self, f'{cb_name}_callback'), event) + sys.monitoring.register_callback(self._tool_id, event, callback) + if event != E.INSTRUCTION: + all_events |= event + self.update_local_events() + sys.monitoring.set_events(self._tool_id, self.GLOBAL_EVENTS) + self._enabled = True + + def stop_trace(self): + self._enabled = False + self._tracing_thread = None + curr_tool = sys.monitoring.get_tool(self._tool_id) + if curr_tool != self._name: + return + sys.monitoring.clear_tool_id(self._tool_id) + sys.monitoring.free_tool_id(self._tool_id) + + def disable_current_event(self): + self._disable_current_event = True + + def restart_events(self): + if sys.monitoring.get_tool(self._tool_id) == self._name: + sys.monitoring.restart_events() + + def callback_wrapper(self, func, event): + import functools + + @functools.wraps(func) + def wrapper(*args): + if self._tracing_thread != threading.current_thread(): + return + try: + frame = sys._getframe().f_back + ret = func(frame, *args) + if self._enabled and frame.f_trace: + self.update_local_events() + if ( + self._disable_current_event + and event not in (E.PY_THROW, E.PY_UNWIND, E.RAISE) + ): + return sys.monitoring.DISABLE + else: + return ret + except BaseException: + self.stop_trace() + sys._getframe().f_back.f_trace = None + raise + finally: + self._disable_current_event = False + + return wrapper + + def call_callback(self, frame, code, *args): + local_tracefunc = self._tracefunc(frame, 'call', None) + if local_tracefunc is not None: + frame.f_trace = local_tracefunc + if self._enabled: + sys.monitoring.set_local_events(self._tool_id, code, self.LOCAL_EVENTS) + + def return_callback(self, frame, code, offset, retval): + if frame.f_trace: + frame.f_trace(frame, 'return', retval) + + def unwind_callback(self, frame, code, *args): + if frame.f_trace: + frame.f_trace(frame, 'return', None) + + def line_callback(self, frame, code, *args): + if frame.f_trace and frame.f_trace_lines: + frame.f_trace(frame, 'line', None) + + def jump_callback(self, frame, code, inst_offset, dest_offset): + if dest_offset > inst_offset: + return sys.monitoring.DISABLE + inst_lineno = self._get_lineno(code, inst_offset) + dest_lineno = self._get_lineno(code, dest_offset) + if inst_lineno != dest_lineno: + return sys.monitoring.DISABLE + if frame.f_trace and frame.f_trace_lines: + frame.f_trace(frame, 'line', None) + + def exception_callback(self, frame, code, offset, exc): + if frame.f_trace: + if exc.__traceback__ and hasattr(exc.__traceback__, 'tb_frame'): + tb = exc.__traceback__ + while tb: + if tb.tb_frame.f_locals.get('self') is self: + return + tb = tb.tb_next + frame.f_trace(frame, 'exception', (type(exc), exc, exc.__traceback__)) + + def opcode_callback(self, frame, code, offset): + if frame.f_trace and frame.f_trace_opcodes: + frame.f_trace(frame, 'opcode', None) + + def update_local_events(self, frame=None): + if sys.monitoring.get_tool(self._tool_id) != self._name: + return + if frame is None: + frame = sys._getframe().f_back + while frame is not None: + if frame.f_trace is not None: + if frame.f_trace_opcodes: + events = self.LOCAL_EVENTS | E.INSTRUCTION + else: + events = self.LOCAL_EVENTS + sys.monitoring.set_local_events(self._tool_id, frame.f_code, events) + frame = frame.f_back + + def _get_lineno(self, code, offset): + import dis + last_lineno = None + for start, lineno in dis.findlinestarts(code): + if offset < start: + return last_lineno + last_lineno = lineno + return last_lineno + + class Bdb: """Generic Python debugger base class. @@ -29,7 +191,7 @@ class Bdb: is determined by the __name__ in the frame globals. """ - def __init__(self, skip=None): + def __init__(self, skip=None, backend='settrace'): self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} @@ -39,6 +201,14 @@ def __init__(self, skip=None): self.enterframe = None self.cmdframe = None self.cmdlineno = None + self.code_linenos = weakref.WeakKeyDictionary() + self.backend = backend + if backend == 'monitoring': + self.monitoring_tracer = _MonitoringTracer() + elif backend == 'settrace': + self.monitoring_tracer = None + else: + raise ValueError(f"Invalid backend '{backend}'") self._load_breaks() @@ -59,6 +229,18 @@ def canonic(self, filename): self.fncache[filename] = canonic return canonic + def start_trace(self): + if self.monitoring_tracer: + self.monitoring_tracer.start_trace(self.trace_dispatch) + else: + sys.settrace(self.trace_dispatch) + + def stop_trace(self): + if self.monitoring_tracer: + self.monitoring_tracer.stop_trace() + else: + sys.settrace(None) + def reset(self): """Set values of attributes as ready to start debugging.""" import linecache @@ -133,7 +315,10 @@ def dispatch_line(self, frame): self.cmdframe == frame and self.cmdlineno == frame.f_lineno ): self.user_line(frame) + self.restart_events() if self.quitting: raise BdbQuit + elif not self.get_break(frame.f_code.co_filename, frame.f_lineno): + self.disable_current_event() return self.trace_dispatch def dispatch_call(self, frame, arg): @@ -149,12 +334,18 @@ def dispatch_call(self, frame, arg): self.botframe = frame.f_back # (CT) Note that this may also be None! return self.trace_dispatch if not (self.stop_here(frame) or self.break_anywhere(frame)): - # No need to trace this function + # We already know there's no breakpoint in this function + # If it's a next/until/return command, we don't need any CALL event + # and we don't need to set the f_trace on any new frame. + # If it's a step command, it must either hit stop_here, or skip the + # whole module. Either way, we don't need the CALL event here. + self.disable_current_event() return # None # Ignore call events in generator except when stepping. if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: return self.trace_dispatch self.user_call(frame, arg) + self.restart_events() if self.quitting: raise BdbQuit return self.trace_dispatch @@ -168,10 +359,14 @@ def dispatch_return(self, frame, arg): if self.stop_here(frame) or frame == self.returnframe: # Ignore return events in generator except when stepping. if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: + # It's possible to trigger a StopIteration exception in + # the caller so we must set the trace function in the caller + self._set_caller_tracefunc(frame) return self.trace_dispatch try: self.frame_returning = frame self.user_return(frame, arg) + self.restart_events() finally: self.frame_returning = None if self.quitting: raise BdbQuit @@ -199,6 +394,7 @@ def dispatch_exception(self, frame, arg): if not (frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS and arg[0] is StopIteration and arg[2] is None): self.user_exception(frame, arg) + self.restart_events() if self.quitting: raise BdbQuit # Stop at the StopIteration or GeneratorExit exception when the user # has set stopframe in a generator by issuing a return command, or a @@ -208,6 +404,7 @@ def dispatch_exception(self, frame, arg): and self.stopframe.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS and arg[0] in (StopIteration, GeneratorExit)): self.user_exception(frame, arg) + self.restart_events() if self.quitting: raise BdbQuit return self.trace_dispatch @@ -217,10 +414,14 @@ def dispatch_opcode(self, frame, arg): If the debugger stops on the current opcode, invoke self.user_opcode(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. + + Opcode event will always trigger the user callback. For now the only + opcode event is from an inline set_trace() and we want to stop there + unconditionally. """ - if self.stop_here(frame) or self.break_here(frame): - self.user_opcode(frame) - if self.quitting: raise BdbQuit + self.user_opcode(frame) + self.restart_events() + if self.quitting: raise BdbQuit return self.trace_dispatch # Normally derived classes don't override the following @@ -286,9 +487,25 @@ def do_clear(self, arg): raise NotImplementedError("subclass of bdb must implement do_clear()") def break_anywhere(self, frame): - """Return True if there is any breakpoint for frame's filename. + """Return True if there is any breakpoint in that frame + """ + filename = self.canonic(frame.f_code.co_filename) + if filename not in self.breaks: + return False + for lineno in self.breaks[filename]: + if self._lineno_in_frame(lineno, frame): + return True + return False + + def _lineno_in_frame(self, lineno, frame): + """Return True if the line number is in the frame's code object. """ - return self.canonic(frame.f_code.co_filename) in self.breaks + code = frame.f_code + if lineno < code.co_firstlineno: + return False + if code not in self.code_linenos: + self.code_linenos[code] = set(lineno for _, _, lineno in code.co_lines()) + return lineno in self.code_linenos[code] # Derived classes should override the user_* methods # to gain control. @@ -322,6 +539,8 @@ def _set_trace_opcodes(self, trace_opcodes): if frame is self.botframe: break frame = frame.f_back + if self.monitoring_tracer: + self.monitoring_tracer.update_local_events() def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, cmdframe=None, cmdlineno=None): @@ -381,7 +600,7 @@ def set_next(self, frame): def set_return(self, frame): """Stop when returning from the given frame.""" if frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: - self._set_stopinfo(frame, None, -1) + self._set_stopinfo(frame, frame, -1) else: self._set_stopinfo(frame.f_back, frame) @@ -390,6 +609,7 @@ def set_trace(self, frame=None): If frame is not specified, debugging starts from caller's frame. """ + self.stop_trace() if frame is None: frame = sys._getframe().f_back self.reset() @@ -402,7 +622,8 @@ def set_trace(self, frame=None): frame.f_trace_lines = True frame = frame.f_back self.set_stepinstr() - sys.settrace(self.trace_dispatch) + self.enterframe = None + self.start_trace() def set_continue(self): """Stop only at breakpoints or when finished. @@ -413,13 +634,15 @@ def set_continue(self): self._set_stopinfo(self.botframe, None, -1) if not self.breaks: # no breakpoints; run without debugger overhead - sys.settrace(None) + self.stop_trace() frame = sys._getframe().f_back while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes + if self.backend == 'monitoring': + self.monitoring_tracer.update_local_events() self.frame_trace_lines_opcodes = {} def set_quit(self): @@ -430,7 +653,7 @@ def set_quit(self): self.stopframe = self.botframe self.returnframe = None self.quitting = True - sys.settrace(None) + self.stop_trace() # Derived classes and clients can call the following methods # to manipulate breakpoints. These methods return an @@ -658,6 +881,16 @@ def format_stack_entry(self, frame_lineno, lprefix=': '): s += f'{lprefix}Warning: lineno is None' return s + def disable_current_event(self): + """Disable the current event.""" + if self.backend == 'monitoring': + self.monitoring_tracer.disable_current_event() + + def restart_events(self): + """Restart all events.""" + if self.backend == 'monitoring': + self.monitoring_tracer.restart_events() + # The following methods can be called by clients to use # a debugger to debug a statement or an expression. # Both can be given as a string, or a code object. @@ -675,14 +908,14 @@ def run(self, cmd, globals=None, locals=None): self.reset() if isinstance(cmd, str): cmd = compile(cmd, "", "exec") - sys.settrace(self.trace_dispatch) + self.start_trace() try: exec(cmd, globals, locals) except BdbQuit: pass finally: self.quitting = True - sys.settrace(None) + self.stop_trace() def runeval(self, expr, globals=None, locals=None): """Debug an expression executed via the eval() function. @@ -695,14 +928,14 @@ def runeval(self, expr, globals=None, locals=None): if locals is None: locals = globals self.reset() - sys.settrace(self.trace_dispatch) + self.start_trace() try: return eval(expr, globals, locals) except BdbQuit: pass finally: self.quitting = True - sys.settrace(None) + self.stop_trace() def runctx(self, cmd, globals, locals): """For backwards-compatibility. Defers to run().""" @@ -717,7 +950,7 @@ def runcall(self, func, /, *args, **kwds): Return the result of the function call. """ self.reset() - sys.settrace(self.trace_dispatch) + self.start_trace() res = None try: res = func(*args, **kwds) @@ -725,7 +958,7 @@ def runcall(self, func, /, *args, **kwds): pass finally: self.quitting = True - sys.settrace(None) + self.stop_trace() return res diff --git a/Lib/code.py b/Lib/code.py index 2777c311187..b134886dc26 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -13,7 +13,6 @@ __all__ = ["InteractiveInterpreter", "InteractiveConsole", "interact", "compile_command"] - class InteractiveInterpreter: """Base class for InteractiveConsole. @@ -126,7 +125,7 @@ def showtraceback(self): """ try: typ, value, tb = sys.exc_info() - self._showtraceback(typ, value, tb.tb_next, '') + self._showtraceback(typ, value, tb.tb_next, "") finally: typ = value = tb = None @@ -140,7 +139,7 @@ def _showtraceback(self, typ, value, tb, source): and not value.text and value.lineno is not None and len(lines) >= value.lineno): value.text = lines[value.lineno - 1] - sys.last_exc = sys.last_value = value = value.with_traceback(tb) + sys.last_exc = sys.last_value = value if sys.excepthook is sys.__excepthook__: self._excepthook(typ, value, tb) else: @@ -220,12 +219,17 @@ def interact(self, banner=None, exitmsg=None): """ try: sys.ps1 + delete_ps1_after = False except AttributeError: sys.ps1 = ">>> " + delete_ps1_after = True try: - sys.ps2 + _ps2 = sys.ps2 + delete_ps2_after = False except AttributeError: sys.ps2 = "... " + delete_ps2_after = True + cprt = 'Type "help", "copyright", "credits" or "license" for more information.' if banner is None: self.write("Python %s on %s\n%s\n(%s)\n" % @@ -288,6 +292,12 @@ def interact(self, banner=None, exitmsg=None): if _quit is not None: builtins.quit = _quit + if delete_ps1_after: + del sys.ps1 + + if delete_ps2_after: + del sys.ps2 + if exitmsg is None: self.write('now exiting %s...\n' % self.__class__.__name__) elif exitmsg != '': @@ -366,7 +376,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=Fa console.raw_input = readfunc else: try: - import readline + import readline # noqa: F401 except ImportError: pass console.interact(banner, exitmsg) @@ -375,9 +385,9 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=Fa if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) parser.add_argument('-q', action='store_true', - help="don't print version and copyright messages") + help="don't print version and copyright messages") args = parser.parse_args() if args.q or sys.flags.quiet: banner = '' diff --git a/Lib/codeop.py b/Lib/codeop.py index adf000ba29f..8cac00442d9 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -47,7 +47,7 @@ PyCF_ONLY_AST = 0x400 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000 -def _maybe_compile(compiler, source, filename, symbol): +def _maybe_compile(compiler, source, filename, symbol, flags): # Check for source consisting of only blank lines and comments. for line in source.split("\n"): line = line.strip() @@ -61,10 +61,10 @@ def _maybe_compile(compiler, source, filename, symbol): with warnings.catch_warnings(): warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning)) try: - compiler(source, filename, symbol) + compiler(source, filename, symbol, flags=flags) except SyntaxError: # Let other compile() errors propagate. try: - compiler(source + "\n", filename, symbol) + compiler(source + "\n", filename, symbol, flags=flags) return None except _IncompleteInputError as e: return None @@ -74,14 +74,13 @@ def _maybe_compile(compiler, source, filename, symbol): return compiler(source, filename, symbol, incomplete_input=False) -def _compile(source, filename, symbol, incomplete_input=True): - flags = 0 +def _compile(source, filename, symbol, incomplete_input=True, *, flags=0): if incomplete_input: flags |= PyCF_ALLOW_INCOMPLETE_INPUT flags |= PyCF_DONT_IMPLY_DEDENT return compile(source, filename, symbol, flags) -def compile_command(source, filename="", symbol="single"): +def compile_command(source, filename="", symbol="single", flags=0): r"""Compile a command and determine whether it is incomplete. Arguments: @@ -100,7 +99,7 @@ def compile_command(source, filename="", symbol="single"): syntax error (OverflowError and ValueError can be produced by malformed literals). """ - return _maybe_compile(_compile, source, filename, symbol) + return _maybe_compile(_compile, source, filename, symbol, flags) class Compile: """Instances of this class behave much like the built-in compile @@ -152,4 +151,4 @@ def __call__(self, source, filename="", symbol="single"): syntax error (OverflowError and ValueError can be produced by malformed literals). """ - return _maybe_compile(self.compiler, source, filename, symbol) + return _maybe_compile(self.compiler, source, filename, symbol, flags=self.compiler.flags) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index db36248c4ff..0aeca1ed7e5 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -57,11 +57,7 @@ try: from _collections import defaultdict except ImportError: - # FIXME: try to implement defaultdict in collections.rs rather than in Python - # I (coolreader18) couldn't figure out some class stuff with __new__ and - # __init__ and __missing__ and subclassing built-in types from Rust, so I went - # with this instead. - from ._defaultdict import defaultdict + pass heapq = None # Lazily imported @@ -557,6 +553,9 @@ class Counter(dict): or multiset. Elements are stored as dictionary keys and their counts are stored as dictionary values. + When constructed from a Mapping or Counter, the original object's + values will be used as the initial counts. + >>> c = Counter('abcdeabcdabcaba') # count elements from a string >>> c.most_common(3) # three most common elements diff --git a/Lib/collections/_defaultdict.py b/Lib/collections/_defaultdict.py deleted file mode 100644 index b9c6c496139..00000000000 --- a/Lib/collections/_defaultdict.py +++ /dev/null @@ -1,58 +0,0 @@ -from reprlib import recursive_repr as _recursive_repr - -class defaultdict(dict): - def __init__(self, *args, **kwargs): - if len(args) >= 1: - default_factory = args[0] - if default_factory is not None and not callable(default_factory): - raise TypeError("first argument must be callable or None") - args = args[1:] - else: - default_factory = None - super().__init__(*args, **kwargs) - self.default_factory = default_factory - - def __missing__(self, key): - if self.default_factory is not None: - val = self.default_factory() - else: - raise KeyError(key) - self[key] = val - return val - - @_recursive_repr() - def __repr_factory(factory): - return repr(factory) - - def __repr__(self): - return f"{type(self).__name__}({defaultdict.__repr_factory(self.default_factory)}, {dict.__repr__(self)})" - - def copy(self): - return type(self)(self.default_factory, self) - - __copy__ = copy - - def __reduce__(self): - if self.default_factory is not None: - args = self.default_factory, - else: - args = () - return type(self), args, None, None, iter(self.items()) - - def __or__(self, other): - if not isinstance(other, dict): - return NotImplemented - - new = defaultdict(self.default_factory, self) - new.update(other) - return new - - def __ror__(self, other): - if not isinstance(other, dict): - return NotImplemented - - new = defaultdict(self.default_factory, other) - new.update(self) - return new - -defaultdict.__module__ = 'collections' diff --git a/Lib/compileall.py b/Lib/compileall.py index 47e2446356e..67fe370451e 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -317,7 +317,9 @@ def main(): import argparse parser = argparse.ArgumentParser( - description='Utilities to support installing Python libraries.') + description='Utilities to support installing Python libraries.', + color=True, + ) parser.add_argument('-l', action='store_const', const=0, default=None, dest='maxlevels', help="don't recurse into subdirectories") diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index 84b25914b0a..5326cf9b19c 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -61,8 +61,9 @@ def __setattr__(self, name, _): def get_frame_info(frame_buffer): """Get Zstandard frame information from a frame header. - *frame_buffer* is a bytes-like object. It should start from the beginning - of a frame, and needs to include at least the frame header (6 to 18 bytes). + *frame_buffer* is a bytes-like object. It should start from the + beginning of a frame, and needs to include at least the frame header + (6 to 18 bytes). The returned FrameInfo object has two attributes. 'decompressed_size' is the size in bytes of the data in the frame when @@ -103,16 +104,17 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level): finalize *zstd_dict* by adding headers and statistics according to the Zstandard dictionary format. - You may compose an effective dictionary content by hand, which is used as - basis dictionary, and use some samples to finalize a dictionary. The basis - dictionary may be a "raw content" dictionary. See *is_raw* in ZstdDict. + You may compose an effective dictionary content by hand, which is used + as basis dictionary, and use some samples to finalize a dictionary. The + basis dictionary may be a "raw content" dictionary. See *is_raw* in + ZstdDict. - *samples* is an iterable of samples, where a sample is a bytes-like object - representing a file. + *samples* is an iterable of samples, where a sample is a bytes-like + object representing a file. *dict_size* is the dictionary's maximum size, in bytes. *level* is the expected compression level. The statistics for each - compression level differ, so tuning the dictionary to the compression level - can provide improvements. + compression level differ, so tuning the dictionary to the compression + level can provide improvements. """ if not isinstance(zstd_dict, ZstdDict): @@ -140,8 +142,8 @@ def compress(data, level=None, options=None, zstd_dict=None): COMPRESSION_LEVEL_DEFAULT ('3'). *options* is a dict object that contains advanced compression parameters. See CompressionParameter for more on options. - *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See - the function train_dict for how to train a ZstdDict on sample data. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. + See the function train_dict for how to train a ZstdDict on sample data. For incremental compression, use a ZstdCompressor instead. """ @@ -152,8 +154,8 @@ def compress(data, level=None, options=None, zstd_dict=None): def decompress(data, zstd_dict=None, options=None): """Decompress one or more frames of Zstandard compressed *data*. - *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See - the function train_dict for how to train a ZstdDict on sample data. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. + See the function train_dict for how to train a ZstdDict on sample data. *options* is a dict object that contains advanced compression parameters. See DecompressionParameter for more on options. diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py index d709f5efc65..26d7c900637 100644 --- a/Lib/compression/zstd/_zstdfile.py +++ b/Lib/compression/zstd/_zstdfile.py @@ -34,11 +34,11 @@ def __init__(self, file, /, mode='r', *, level=None, options=None, zstd_dict=None): """Open a Zstandard compressed file in binary mode. - *file* can be either an file-like object, or a file name to open. + *file* can be either a file-like object, or a file name to open. - *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' for - creating exclusively, or 'a' for appending. These can equivalently be - given as 'rb', 'wb', 'xb' and 'ab' respectively. + *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' + for creating exclusively, or 'a' for appending. These can + equivalently be given as 'rb', 'wb', 'xb' and 'ab' respectively. *level* is an optional int specifying the compression level to use, or COMPRESSION_LEVEL_DEFAULT if not given. @@ -246,6 +246,7 @@ def peek(self, size=-1): return self._buffer.peek(size) def __next__(self): + self._check_can_read() if ret := self._buffer.readline(): return ret raise StopIteration @@ -296,26 +297,27 @@ def open(file, /, mode='rb', *, level=None, options=None, zstd_dict=None, encoding=None, errors=None, newline=None): """Open a Zstandard compressed file in binary or text mode. - file can be either a file name (given as a str, bytes, or PathLike object), - in which case the named file is opened, or it can be an existing file object - to read from or write to. + file can be either a file name (given as a str, bytes, or PathLike + object), in which case the named file is opened, or it can be + an existing file object to read from or write to. - The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', 'a', - 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. + The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', + 'a', 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. - The level, options, and zstd_dict parameters specify the settings the same - as ZstdFile. + The level, options, and zstd_dict parameters specify the settings the + same as ZstdFile. When using read mode (decompression), the options parameter is a dict - representing advanced decompression options. The level parameter is not - supported in this case. When using write mode (compression), only one of - level, an int representing the compression level, or options, a dict - representing advanced compression options, may be passed. In both modes, - zstd_dict is a ZstdDict instance containing a trained Zstandard dictionary. - - For binary mode, this function is equivalent to the ZstdFile constructor: - ZstdFile(filename, mode, ...). In this case, the encoding, errors and - newline parameters must not be provided. + representing advanced decompression options. The level parameter is not + supported in this case. When using write mode (compression), only one + of level, an int representing the compression level, or options, a dict + representing advanced compression options, may be passed. In both + modes, zstd_dict is a ZstdDict instance containing a trained Zstandard + dictionary. + + For binary mode, this function is equivalent to the ZstdFile + constructor: ZstdFile(filename, mode, ...). In this case, the encoding, + errors and newline parameters must not be provided. For text mode, an ZstdFile object is created, and wrapped in an io.TextIOWrapper instance with the specified encoding, error handling diff --git a/Lib/concurrent/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py new file mode 100644 index 00000000000..ea4147ee9a2 --- /dev/null +++ b/Lib/concurrent/interpreters/__init__.py @@ -0,0 +1,247 @@ +"""Subinterpreters High Level Module.""" + +import threading +import weakref +import _interpreters + +# aliases: +from _interpreters import ( + InterpreterError, InterpreterNotFoundError, NotShareableError, + is_shareable, +) +from ._queues import ( + create as create_queue, + Queue, QueueEmpty, QueueFull, +) + + +__all__ = [ + 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', + 'Interpreter', + 'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed', + 'NotShareableError', + 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull', +] + + +_EXEC_FAILURE_STR = """ +{superstr} + +Uncaught in the interpreter: + +{formatted} +""".strip() + +class ExecutionFailed(InterpreterError): + """An unhandled exception happened during execution. + + This is raised from Interpreter.exec() and Interpreter.call(). + """ + + def __init__(self, excinfo): + msg = excinfo.formatted + if not msg: + if excinfo.type and excinfo.msg: + msg = f'{excinfo.type.__name__}: {excinfo.msg}' + else: + msg = excinfo.type.__name__ or excinfo.msg + super().__init__(msg) + self.excinfo = excinfo + + def __str__(self): + try: + formatted = self.excinfo.errdisplay + except Exception: + return super().__str__() + else: + return _EXEC_FAILURE_STR.format( + superstr=super().__str__(), + formatted=formatted, + ) + + +def create(): + """Return a new (idle) Python interpreter.""" + id = _interpreters.create(reqrefs=True) + return Interpreter(id, _ownsref=True) + + +def list_all(): + """Return all existing interpreters.""" + return [Interpreter(id, _whence=whence) + for id, whence in _interpreters.list_all(require_ready=True)] + + +def get_current(): + """Return the currently running interpreter.""" + id, whence = _interpreters.get_current() + return Interpreter(id, _whence=whence) + + +def get_main(): + """Return the main interpreter.""" + id, whence = _interpreters.get_main() + assert whence == _interpreters.WHENCE_RUNTIME, repr(whence) + return Interpreter(id, _whence=whence) + + +_known = weakref.WeakValueDictionary() + +class Interpreter: + """A single Python interpreter. + + Attributes: + + "id" - the unique process-global ID number for the interpreter + "whence" - indicates where the interpreter was created + + If the interpreter wasn't created by this module + then any method that modifies the interpreter will fail, + i.e. .close(), .prepare_main(), .exec(), and .call() + """ + + _WHENCE_TO_STR = { + _interpreters.WHENCE_UNKNOWN: 'unknown', + _interpreters.WHENCE_RUNTIME: 'runtime init', + _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', + _interpreters.WHENCE_CAPI: 'C-API', + _interpreters.WHENCE_XI: 'cross-interpreter C-API', + _interpreters.WHENCE_STDLIB: '_interpreters module', + } + + def __new__(cls, id, /, _whence=None, _ownsref=None): + # There is only one instance for any given ID. + if not isinstance(id, int): + raise TypeError(f'id must be an int, got {id!r}') + id = int(id) + if _whence is None: + if _ownsref: + _whence = _interpreters.WHENCE_STDLIB + else: + _whence = _interpreters.whence(id) + assert _whence in cls._WHENCE_TO_STR, repr(_whence) + if _ownsref is None: + _ownsref = (_whence == _interpreters.WHENCE_STDLIB) + try: + self = _known[id] + assert hasattr(self, '_ownsref') + except KeyError: + self = super().__new__(cls) + _known[id] = self + self._id = id + self._whence = _whence + self._ownsref = _ownsref + if _ownsref: + # This may raise InterpreterNotFoundError: + _interpreters.incref(id) + return self + + def __repr__(self): + return f'{type(self).__name__}({self.id})' + + def __hash__(self): + return hash(self._id) + + def __del__(self): + self._decref() + + # for pickling: + def __reduce__(self): + return (type(self), (self._id,)) + + # gh-135729: Globals might be destroyed by the time this is called, so we + # need to keep references ourself + def _decref(self, *, + InterpreterNotFoundError=InterpreterNotFoundError, + _interp_decref=_interpreters.decref, + ): + if not self._ownsref: + return + self._ownsref = False + try: + _interp_decref(self._id) + except InterpreterNotFoundError: + pass + + @property + def id(self): + return self._id + + @property + def whence(self): + return self._WHENCE_TO_STR[self._whence] + + def is_running(self): + """Return whether or not the identified interpreter is running.""" + return _interpreters.is_running(self._id) + + # Everything past here is available only to interpreters created by + # interpreters.create(). + + def close(self): + """Finalize and destroy the interpreter. + + Attempting to destroy the current interpreter results + in an InterpreterError. + """ + return _interpreters.destroy(self._id, restrict=True) + + def prepare_main(self, ns=None, /, **kwargs): + """Bind the given values into the interpreter's __main__. + + The values must be shareable. + """ + ns = dict(ns, **kwargs) if ns is not None else kwargs + _interpreters.set___main___attrs(self._id, ns, restrict=True) + + def exec(self, code, /): + """Run the given source code in the interpreter. + + This is essentially the same as calling the builtin "exec" + with this interpreter, using the __dict__ of its __main__ + module as both globals and locals. + + There is no return value. + + If the code raises an unhandled exception then an ExecutionFailed + exception is raised, which summarizes the unhandled exception. + The actual exception is discarded because objects cannot be + shared between interpreters. + + This blocks the current Python thread until done. During + that time, the previous interpreter is allowed to run + in other threads. + """ + excinfo = _interpreters.exec(self._id, code, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + + def _call(self, callable, args, kwargs): + res, excinfo = _interpreters.call(self._id, callable, args, kwargs, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + return res + + def call(self, callable, /, *args, **kwargs): + """Call the object in the interpreter with given args/kwargs. + + Nearly all callables, args, kwargs, and return values are + supported. All "shareable" objects are supported, as are + "stateless" functions (meaning non-closures that do not use + any globals). This method will fall back to pickle. + + If the callable raises an exception then the error display + (including full traceback) is sent back between the interpreters + and an ExecutionFailed exception is raised, much like what + happens with Interpreter.exec(). + """ + return self._call(callable, args, kwargs) + + def call_in_thread(self, callable, /, *args, **kwargs): + """Return a new thread that calls the object in the interpreter. + + The return value and any raised exception are discarded. + """ + t = threading.Thread(target=self._call, args=(callable, args, kwargs)) + t.start() + return t diff --git a/Lib/concurrent/interpreters/_crossinterp.py b/Lib/concurrent/interpreters/_crossinterp.py new file mode 100644 index 00000000000..a5f46b20fbb --- /dev/null +++ b/Lib/concurrent/interpreters/_crossinterp.py @@ -0,0 +1,107 @@ +"""Common code between queues and channels.""" + + +class ItemInterpreterDestroyed(Exception): + """Raised when trying to get an item whose interpreter was destroyed.""" + + +class classonly: + """A non-data descriptor that makes a value only visible on the class. + + This is like the "classmethod" builtin, but does not show up on + instances of the class. It may be used as a decorator. + """ + + def __init__(self, value): + self.value = value + self.getter = classmethod(value).__get__ + self.name = None + + def __set_name__(self, cls, name): + if self.name is not None: + raise TypeError('already used') + self.name = name + + def __get__(self, obj, cls): + if obj is not None: + raise AttributeError(self.name) + # called on the class + return self.getter(None, cls) + + +class UnboundItem: + """Represents a cross-interpreter item no longer bound to an interpreter. + + An item is unbound when the interpreter that added it to the + cross-interpreter container is destroyed. + """ + + __slots__ = () + + @classonly + def singleton(cls, kind, module, name='UNBOUND'): + doc = cls.__doc__ + if doc: + doc = doc.replace( + 'cross-interpreter container', kind, + ).replace( + 'cross-interpreter', kind, + ) + subclass = type( + f'Unbound{kind.capitalize()}Item', + (cls,), + { + "_MODULE": module, + "_NAME": name, + "__doc__": doc, + }, + ) + return object.__new__(subclass) + + _MODULE = __name__ + _NAME = 'UNBOUND' + + def __new__(cls): + raise Exception(f'use {cls._MODULE}.{cls._NAME}') + + def __repr__(self): + return f'{self._MODULE}.{self._NAME}' +# return f'interpreters._queues.UNBOUND' + + +UNBOUND = object.__new__(UnboundItem) +UNBOUND_ERROR = object() +UNBOUND_REMOVE = object() + +_UNBOUND_CONSTANT_TO_FLAG = { + UNBOUND_REMOVE: 1, + UNBOUND_ERROR: 2, + UNBOUND: 3, +} +_UNBOUND_FLAG_TO_CONSTANT = {v: k + for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()} + + +def serialize_unbound(unbound): + op = unbound + try: + flag = _UNBOUND_CONSTANT_TO_FLAG[op] + except KeyError: + raise NotImplementedError(f'unsupported unbound replacement op {op!r}') + return flag, + + +def resolve_unbound(flag, exctype_destroyed): + try: + op = _UNBOUND_FLAG_TO_CONSTANT[flag] + except KeyError: + raise NotImplementedError(f'unsupported unbound replacement op {flag!r}') + if op is UNBOUND_REMOVE: + # "remove" not possible here + raise NotImplementedError + elif op is UNBOUND_ERROR: + raise exctype_destroyed("item's original interpreter destroyed") + elif op is UNBOUND: + return UNBOUND + else: + raise NotImplementedError(repr(op)) diff --git a/Lib/concurrent/interpreters/_queues.py b/Lib/concurrent/interpreters/_queues.py new file mode 100644 index 00000000000..56c947f7d2a --- /dev/null +++ b/Lib/concurrent/interpreters/_queues.py @@ -0,0 +1,290 @@ +"""Cross-interpreter Queues High Level Module.""" + +import pickle +import queue +import time +import weakref +import _interpqueues as _queues +from . import _crossinterp + +# aliases: +from _interpqueues import ( + QueueError, QueueNotFoundError, +) +from ._crossinterp import ( + UNBOUND_ERROR, UNBOUND_REMOVE, +) + +__all__ = [ + 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', + 'create', 'list_all', + 'Queue', + 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull', + 'ItemInterpreterDestroyed', +] + + +class QueueEmpty(QueueError, queue.Empty): + """Raised from get_nowait() when the queue is empty. + + It is also raised from get() if it times out. + """ + + +class QueueFull(QueueError, queue.Full): + """Raised from put_nowait() when the queue is full. + + It is also raised from put() if it times out. + """ + + +class ItemInterpreterDestroyed(QueueError, + _crossinterp.ItemInterpreterDestroyed): + """Raised from get() and get_nowait().""" + + +_SHARED_ONLY = 0 +_PICKLED = 1 + + +UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__) + + +def _serialize_unbound(unbound): + if unbound is UNBOUND: + unbound = _crossinterp.UNBOUND + return _crossinterp.serialize_unbound(unbound) + + +def _resolve_unbound(flag): + resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed) + if resolved is _crossinterp.UNBOUND: + resolved = UNBOUND + return resolved + + +def create(maxsize=0, *, unbounditems=UNBOUND): + """Return a new cross-interpreter queue. + + The queue may be used to pass data safely between interpreters. + + "unbounditems" sets the default for Queue.put(); see that method for + supported values. The default value is UNBOUND, which replaces + the unbound item. + """ + unbound = _serialize_unbound(unbounditems) + unboundop, = unbound + qid = _queues.create(maxsize, unboundop, -1) + self = Queue(qid) + self._set_unbound(unboundop, unbounditems) + return self + + +def list_all(): + """Return a list of all open queues.""" + queues = [] + for qid, unboundop, _ in _queues.list_all(): + self = Queue(qid) + if not hasattr(self, '_unbound'): + self._set_unbound(unboundop) + else: + assert self._unbound[0] == unboundop + queues.append(self) + return queues + + +_known_queues = weakref.WeakValueDictionary() + +class Queue: + """A cross-interpreter queue.""" + + def __new__(cls, id, /): + # There is only one instance for any given ID. + if isinstance(id, int): + id = int(id) + else: + raise TypeError(f'id must be an int, got {id!r}') + try: + self = _known_queues[id] + except KeyError: + self = super().__new__(cls) + self._id = id + _known_queues[id] = self + _queues.bind(id) + return self + + def __del__(self): + try: + _queues.release(self._id) + except QueueNotFoundError: + pass + try: + del _known_queues[self._id] + except KeyError: + pass + + def __repr__(self): + return f'{type(self).__name__}({self.id})' + + def __hash__(self): + return hash(self._id) + + # for pickling: + def __reduce__(self): + return (type(self), (self._id,)) + + def _set_unbound(self, op, items=None): + assert not hasattr(self, '_unbound') + if items is None: + items = _resolve_unbound(op) + unbound = (op, items) + self._unbound = unbound + return unbound + + @property + def id(self): + return self._id + + @property + def unbounditems(self): + try: + _, items = self._unbound + except AttributeError: + op, _ = _queues.get_queue_defaults(self._id) + _, items = self._set_unbound(op) + return items + + @property + def maxsize(self): + try: + return self._maxsize + except AttributeError: + self._maxsize = _queues.get_maxsize(self._id) + return self._maxsize + + def empty(self): + return self.qsize() == 0 + + def full(self): + return _queues.is_full(self._id) + + def qsize(self): + return _queues.get_count(self._id) + + def put(self, obj, block=True, timeout=None, *, + unbounditems=None, + _delay=10 / 1000, # 10 milliseconds + ): + """Add the object to the queue. + + If "block" is true, this blocks while the queue is full. + + For most objects, the object received through Queue.get() will + be a new one, equivalent to the original and not sharing any + actual underlying data. The notable exceptions include + cross-interpreter types (like Queue) and memoryview, where the + underlying data is actually shared. Furthermore, some types + can be sent through a queue more efficiently than others. This + group includes various immutable types like int, str, bytes, and + tuple (if the items are likewise efficiently shareable). + See interpreters.is_shareable(). + + "unbounditems" controls the behavior of Queue.get() for the given + object if the current interpreter (calling put()) is later + destroyed. + + If "unbounditems" is None (the default) then it uses the + queue's default, set with create_queue(), + which is usually UNBOUND. + + If "unbounditems" is UNBOUND_ERROR then get() will raise an + ItemInterpreterDestroyed exception if the original interpreter + has been destroyed. This does not otherwise affect the queue; + the next call to put() will work like normal, returning the next + item in the queue. + + If "unbounditems" is UNBOUND_REMOVE then the item will be removed + from the queue as soon as the original interpreter is destroyed. + Be aware that this will introduce an imbalance between put() + and get() calls. + + If "unbounditems" is UNBOUND then it is returned by get() in place + of the unbound item. + """ + if not block: + return self.put_nowait(obj, unbounditems=unbounditems) + if unbounditems is None: + unboundop = -1 + else: + unboundop, = _serialize_unbound(unbounditems) + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + while True: + try: + _queues.put(self._id, obj, unboundop) + except QueueFull as exc: + if timeout is not None and time.time() >= end: + raise # re-raise + time.sleep(_delay) + else: + break + + def put_nowait(self, obj, *, unbounditems=None): + if unbounditems is None: + unboundop = -1 + else: + unboundop, = _serialize_unbound(unbounditems) + _queues.put(self._id, obj, unboundop) + + def get(self, block=True, timeout=None, *, + _delay=10 / 1000, # 10 milliseconds + ): + """Return the next object from the queue. + + If "block" is true, this blocks while the queue is empty. + + If the next item's original interpreter has been destroyed + then the "next object" is determined by the value of the + "unbounditems" argument to put(). + """ + if not block: + return self.get_nowait() + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + while True: + try: + obj, unboundop = _queues.get(self._id) + except QueueEmpty as exc: + if timeout is not None and time.time() >= end: + raise # re-raise + time.sleep(_delay) + else: + break + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + return obj + + def get_nowait(self): + """Return the next object from the channel. + + If the queue is empty then raise QueueEmpty. Otherwise this + is the same as get(). + """ + try: + obj, unboundop = _queues.get(self._id) + except QueueEmpty as exc: + raise # re-raise + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + return obj + + +_queues._register_heap_types(Queue, QueueEmpty, QueueFull) diff --git a/Lib/configparser.py b/Lib/configparser.py index 05b86acb919..3c452afe8ad 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -154,14 +154,13 @@ import os import re import sys -import types __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", - "MultilineContinuationError", - "ConfigParser", "RawConfigParser", + "MultilineContinuationError", "UnnamedSectionDisabledError", + "InvalidWriteError", "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", "SectionProxy", "ConverterMapping", "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION") @@ -316,12 +315,15 @@ def __init__(self, source, *args): def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + self.message += f'\n\t[line {lineno:2d}]: {line!r}' def combine(self, others): + messages = [self.message] for other in others: - for error in other.errors: - self.append(*error) + for lineno, line in other.errors: + self.errors.append((lineno, line)) + messages.append(f'\n\t[line {lineno:2d}]: {line!r}') + self.message = "".join(messages) return self @staticmethod @@ -362,11 +364,27 @@ def __init__(self, filename, lineno, line): self.line = line self.args = (filename, lineno, line) + +class UnnamedSectionDisabledError(Error): + """Raised when an attempt to use UNNAMED_SECTION is made with the + feature disabled.""" + def __init__(self): + Error.__init__(self, "Support for UNNAMED_SECTION is disabled.") + + class _UnnamedSection: def __repr__(self): return "" +class InvalidWriteError(Error): + """Raised when attempting to write data that the parser would read back differently. + ex: writing a key which begins with the section header pattern would read back as a + new section """ + + def __init__(self, msg=''): + Error.__init__(self, msg) + UNNAMED_SECTION = _UnnamedSection() @@ -556,35 +574,36 @@ def __init__(self): class _Line(str): + __slots__ = 'clean', 'has_comments' def __new__(cls, val, *args, **kwargs): return super().__new__(cls, val) - def __init__(self, val, prefixes): - self.prefixes = prefixes - - @functools.cached_property - def clean(self): - return self._strip_full() and self._strip_inline() + def __init__(self, val, comments): + trimmed = val.strip() + self.clean = comments.strip(trimmed) + self.has_comments = trimmed != self.clean - @property - def has_comments(self): - return self.strip() != self.clean - def _strip_inline(self): - """ - Search for the earliest prefix at the beginning of the line or following a space. - """ - matcher = re.compile( - '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline) - # match nothing if no prefixes - or '(?!)' +class _CommentSpec: + def __init__(self, full_prefixes, inline_prefixes): + full_patterns = ( + # prefix at the beginning of a line + fr'^({re.escape(prefix)}).*' + for prefix in full_prefixes ) - match = matcher.search(self) - return self[:match.start() if match else None].strip() + inline_patterns = ( + # prefix at the beginning of the line or following a space + fr'(^|\s)({re.escape(prefix)}.*)' + for prefix in inline_prefixes + ) + self.pattern = re.compile('|'.join(itertools.chain(full_patterns, inline_patterns))) + + def strip(self, text): + return self.pattern.sub('', text).rstrip() - def _strip_full(self): - return '' if any(map(self.strip().startswith, self.prefixes.full)) else True + def wrap(self, text): + return _Line(text, self) class RawConfigParser(MutableMapping): @@ -597,7 +616,9 @@ class RawConfigParser(MutableMapping): \] # ] """ _OPT_TMPL = r""" - (?P