fix(strings): count and slice strings by Unicode code point
str_length and str_slice used UTF-8 byte offsets, so any input with
multi-byte characters produced wrong results: str_length("cafe") with
an accented final character returned 5 instead of 4, and str_slice(0,
4, ...) on the same string returned an empty string because the byte
index fell inside the encoding of the last character. This also broke
the higher-level helpers built on top of them (str_find, str_contains,
str_replace).
Count characters with chars().count() and index str_slice by character.
Out-of-range slices still return an empty string, so behavior for ASCII
input is unchanged.
This commit is contained in:
1 parent
62dfa47710
commit
8a9a70d15e
2 files changed
+19
-3
No files matched your search
@@ -1,11 +1,19 @@
|
||||
assert_eq(str_length(""), 0)
|
||||
assert_eq(str_length("foo"), 3)
|
||||
|
||||
# Length and slicing operate on Unicode code points, not on UTF-8 bytes
|
||||
assert_eq(str_length("café"), 4)
|
||||
assert_eq(str_length("❤"), 1)
|
||||
|
||||
assert_eq(str_slice(0, 5, "hello world"), "hello")
|
||||
assert_eq(str_slice(6, 11, "hello world"), "world")
|
||||
assert_eq(str_slice(0, 0, "hello world"), "")
|
||||
assert_eq(str_slice(0, 100, "hello world"), "")
|
||||
|
||||
assert_eq(str_slice(0, 4, "café"), "café")
|
||||
assert_eq(str_slice(3, 4, "café"), "é")
|
||||
assert_eq(str_slice(0, 3, "café"), "caf")
|
||||
|
||||
assert_eq(chr(65), "A")
|
||||
assert_eq(chr(97), "a")
|
||||
assert_eq(chr(0x2764), "❤")
|
||||
|
||||
@@ -12,7 +12,7 @@ pub fn str_length(
|
||||
mut args: Args,
|
||||
_return_type: &TypeScheme,
|
||||
) -> Result<Value, Box<RuntimeErrorKind>> {
|
||||
let len = string_arg!(args).len();
|
||||
let len = string_arg!(args).chars().count();
|
||||
return_scalar!(len as f64)
|
||||
}
|
||||
|
||||
@@ -41,9 +41,17 @@ pub fn str_slice(
|
||||
let end = quantity_arg!(args).unsafe_value().to_f64() as usize;
|
||||
let input = string_arg!(args);
|
||||
|
||||
let output = input.get(start..end).unwrap_or_default();
|
||||
// Indices refer to Unicode code points, not bytes, so that slicing does not
|
||||
// depend on the UTF-8 encoding of the input. Out-of-range requests yield an
|
||||
// empty string, matching the previous byte-based behavior.
|
||||
let characters = input.chars().collect::<Vec<char>>();
|
||||
let output = if start <= end && end <= characters.len() {
|
||||
characters[start..end].iter().collect::<String>()
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
return_string!(borrowed = output)
|
||||
return_string!(from = &output)
|
||||
}
|
||||
|
||||
pub fn chr(
|
||||
|
||||
Reference in new issue
Block a user