Skip to content

Commit 0b1b4f9

Browse files
joshuamegnauth54joshuamegnauth54
authored
Use Unicode properties for alnum, alpha, etc. (#7626)
Rust and Python differ in which properties they use for alphanumeric, numeric, et cetera. Both languages list which properties are used which makes it easy to mimic Python's behavior in Rust. My previous patch was a bit shortsighted because I filtered out combining characters from is_alphanumeric. Using properties is exact and also much cleaner. It also covers edge cases that my initial approach missed. Besides isalnum, I also fixed isnumeric and isdigit in the same way by using properties.
1 parent bbe994e commit 0b1b4f9

4 files changed

Lines changed: 36 additions & 14 deletions

File tree

‎Lib/test/test_str.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -792,7 +792,6 @@ def test_isdecimal(self):
792792
for ch in ['\U0001D7F6', '\U00011066', '\U000104A0']:
793793
self.assertTrue(ch.isdecimal(), '{!a} is decimal.'.format(ch))
794794

795-
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False != True
796795
def test_isdigit(self):
797796
super().test_isdigit()
798797
self.checkequalnofix(True, '\u2460', 'isdigit')

‎crates/sre_engine/src/string.rs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use icu_properties::props::{CanonicalCombiningClass, EnumeratedProperty};
1+
use icu_properties::props::{EnumeratedProperty, GeneralCategory, GeneralCategoryGroup};
22
use rustpython_wtf8::Wtf8;
33

44
#[derive(Debug, Clone, Copy)]
@@ -444,9 +444,10 @@ pub(crate) const fn is_uni_linebreak(ch: u32) -> bool {
444444
pub(crate) fn is_uni_alnum(ch: u32) -> bool {
445445
// TODO: check with cpython
446446
char::try_from(ch)
447-
.map(|x| {
448-
x.is_alphanumeric()
449-
&& CanonicalCombiningClass::for_char(x) == CanonicalCombiningClass::NotReordered
447+
.map(|c| {
448+
GeneralCategoryGroup::Letter
449+
.union(GeneralCategoryGroup::Number)
450+
.contains(GeneralCategory::for_char(c))
450451
})
451452
.unwrap_or(false)
452453
}

‎crates/vm/src/builtins/str.rs‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ use rustpython_common::{
4545
};
4646

4747
use icu_properties::props::{
48-
BidiClass, BinaryProperty, CanonicalCombiningClass, EnumeratedProperty, GeneralCategory,
49-
XidContinue, XidStart,
48+
BidiClass, BinaryProperty, EnumeratedProperty, GeneralCategory, GeneralCategoryGroup,
49+
NumericType, XidContinue, XidStart,
5050
};
5151
use unicode_casing::CharExt;
5252

@@ -949,23 +949,30 @@ impl PyStr {
949949
fn isalnum(&self) -> bool {
950950
!self.data.is_empty()
951951
&& self.char_all(|c| {
952-
c.is_alphanumeric()
953-
&& CanonicalCombiningClass::for_char(c) == CanonicalCombiningClass::NotReordered
952+
GeneralCategoryGroup::Letter
953+
.union(GeneralCategoryGroup::Number)
954+
.contains(GeneralCategory::for_char(c))
954955
})
955956
}
956957

957958
#[pymethod]
958959
fn isnumeric(&self) -> bool {
959-
!self.data.is_empty() && self.char_all(char::is_numeric)
960+
!self.data.is_empty()
961+
&& self.char_all(|c| {
962+
[
963+
NumericType::Decimal,
964+
NumericType::Digit,
965+
NumericType::Numeric,
966+
]
967+
.contains(&NumericType::for_char(c))
968+
})
960969
}
961970

962971
#[pymethod]
963972
fn isdigit(&self) -> bool {
964-
// python's isdigit also checks if exponents are digits, these are the unicode codepoints for exponents
965973
!self.data.is_empty()
966974
&& self.char_all(|c| {
967-
c.is_ascii_digit()
968-
|| matches!(c, '⁰' | '¹' | '²' | '³' | '⁴' | '⁵' | '⁶' | '⁷' | '⁸' | '⁹')
975+
[NumericType::Digit, NumericType::Decimal].contains(&NumericType::for_char(c))
969976
})
970977
}
971978

@@ -1064,7 +1071,9 @@ impl PyStr {
10641071

10651072
#[pymethod]
10661073
fn isalpha(&self) -> bool {
1067-
!self.data.is_empty() && self.char_all(char::is_alphabetic)
1074+
!self.data.is_empty()
1075+
&& self
1076+
.char_all(|c| GeneralCategoryGroup::Letter.contains(GeneralCategory::for_char(c)))
10681077
}
10691078

10701079
#[pymethod]

‎extra_tests/snippets/builtin_str.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,16 +72,29 @@
7272
assert "\u1c89".istitle()
7373
# assert "DZ".title() == "Dz"
7474
assert a.isalpha()
75+
assert not "\u093f".isalpha()
7576

7677
# Combining characters differ slightly between Rust and Python
7778
assert "\u006e".isalnum()
7879
assert not "\u0303".isalnum()
7980
assert not "\u006e\u0303".isalnum()
8081
assert "\u00f1".isalnum()
8182
assert not "\u0345".isalnum()
83+
assert not "\u093f".isalnum()
8284
for raw in range(0x0363, 0x036F):
8385
assert not chr(raw).isalnum()
8486

87+
# isdigit is true for exponents
88+
assert "⁰".isdigit()
89+
assert "⁰".isnumeric()
90+
assert not "½".isdigit()
91+
assert "½".isnumeric()
92+
assert not "Ⅻ".isdigit()
93+
assert "Ⅻ".isnumeric()
94+
95+
# isnumeric is broader than Rust's
96+
assert "\u3405".isnumeric()
97+
8598
s = "1 2 3"
8699
assert s.split(" ", 1) == ["1", "2 3"]
87100
assert s.rsplit(" ", 1) == ["1 2", "3"]

0 commit comments

Comments
 (0)