fix(strings): count and slice strings by Unicode code point
str_length and str_slice used UTF-8 byte offsets, so any input with
multi-byte characters produced wrong results: str_length("cafe") with
an accented final character returned 5 instead of 4, and str_slice(0,
4, ...) on the same string returned an empty string because the byte
index fell inside the encoding of the last character. This also broke
the higher-level helpers built on top of them (str_find, str_contains,
str_replace).
Count characters with chars().count() and index str_slice by character.
Out-of-range slices still return an empty string, so behavior for ASCII
input is unchanged.
This commit is contained in:
1 parent
62dfa47710
commit
8a9a70d15e
2 files changed
+19
-3
No files matched your search
@@ -1,11 +1,19 @@
|
|||||||
assert_eq(str_length(""), 0)
|
assert_eq(str_length(""), 0)
|
||||||
assert_eq(str_length("foo"), 3)
|
assert_eq(str_length("foo"), 3)
|
||||||
|
|
||||||
|
# Length and slicing operate on Unicode code points, not on UTF-8 bytes
|
||||||
|
assert_eq(str_length("café"), 4)
|
||||||
|
assert_eq(str_length("❤"), 1)
|
||||||
|
|
||||||
assert_eq(str_slice(0, 5, "hello world"), "hello")
|
assert_eq(str_slice(0, 5, "hello world"), "hello")
|
||||||
assert_eq(str_slice(6, 11, "hello world"), "world")
|
assert_eq(str_slice(6, 11, "hello world"), "world")
|
||||||
assert_eq(str_slice(0, 0, "hello world"), "")
|
assert_eq(str_slice(0, 0, "hello world"), "")
|
||||||
assert_eq(str_slice(0, 100, "hello world"), "")
|
assert_eq(str_slice(0, 100, "hello world"), "")
|
||||||
|
|
||||||
|
assert_eq(str_slice(0, 4, "café"), "café")
|
||||||
|
assert_eq(str_slice(3, 4, "café"), "é")
|
||||||
|
assert_eq(str_slice(0, 3, "café"), "caf")
|
||||||
|
|
||||||
assert_eq(chr(65), "A")
|
assert_eq(chr(65), "A")
|
||||||
assert_eq(chr(97), "a")
|
assert_eq(chr(97), "a")
|
||||||
assert_eq(chr(0x2764), "❤")
|
assert_eq(chr(0x2764), "❤")
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ pub fn str_length(
|
|||||||
mut args: Args,
|
mut args: Args,
|
||||||
_return_type: &TypeScheme,
|
_return_type: &TypeScheme,
|
||||||
) -> Result<Value, Box<RuntimeErrorKind>> {
|
) -> Result<Value, Box<RuntimeErrorKind>> {
|
||||||
let len = string_arg!(args).len();
|
let len = string_arg!(args).chars().count();
|
||||||
return_scalar!(len as f64)
|
return_scalar!(len as f64)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,9 +41,17 @@ pub fn str_slice(
|
|||||||
let end = quantity_arg!(args).unsafe_value().to_f64() as usize;
|
let end = quantity_arg!(args).unsafe_value().to_f64() as usize;
|
||||||
let input = string_arg!(args);
|
let input = string_arg!(args);
|
||||||
|
|
||||||
let output = input.get(start..end).unwrap_or_default();
|
// Indices refer to Unicode code points, not bytes, so that slicing does not
|
||||||
|
// depend on the UTF-8 encoding of the input. Out-of-range requests yield an
|
||||||
|
// empty string, matching the previous byte-based behavior.
|
||||||
|
let characters = input.chars().collect::<Vec<char>>();
|
||||||
|
let output = if start <= end && end <= characters.len() {
|
||||||
|
characters[start..end].iter().collect::<String>()
|
||||||
|
} else {
|
||||||
|
String::new()
|
||||||
|
};
|
||||||
|
|
||||||
return_string!(borrowed = output)
|
return_string!(from = &output)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn chr(
|
pub fn chr(
|
||||||
|
|||||||
Reference in new issue
Block a user