fix(strings): count and slice strings by Unicode code point

str_length and str_slice used UTF-8 byte offsets, so any input with
multi-byte characters produced wrong results: str_length("cafe") with
an accented final character returned 5 instead of 4, and str_slice(0,
4, ...) on the same string returned an empty string because the byte
index fell inside the encoding of the last character. This also broke
the higher-level helpers built on top of them (str_find, str_contains,
str_replace).

Count characters with chars().count() and index str_slice by character.
Out-of-range slices still return an empty string, so behavior for ASCII
input is unchanged.
This commit is contained in:
dualfroz committed 2026-09-02 10:25:31 +02:00
1 parent 62dfa47710
commit 8a9a70d15e
2 files changed
+19 -3

No files matched your search

+8
View File
@@ -1,11 +1,19 @@
assert_eq(str_length(""), 0)
assert_eq(str_length("foo"), 3)
# Length and slicing operate on Unicode code points, not on UTF-8 bytes
assert_eq(str_length("café"), 4)
assert_eq(str_length("❤"), 1)
assert_eq(str_slice(0, 5, "hello world"), "hello")
assert_eq(str_slice(6, 11, "hello world"), "world")
assert_eq(str_slice(0, 0, "hello world"), "")
assert_eq(str_slice(0, 100, "hello world"), "")
assert_eq(str_slice(0, 4, "café"), "café")
assert_eq(str_slice(3, 4, "café"), "é")
assert_eq(str_slice(0, 3, "café"), "caf")
assert_eq(chr(65), "A")
assert_eq(chr(97), "a")
assert_eq(chr(0x2764), "❤")
+11 -3
View File
@@ -12,7 +12,7 @@ pub fn str_length(
mut args: Args,
_return_type: &TypeScheme,
) -> Result<Value, Box<RuntimeErrorKind>> {
let len = string_arg!(args).len();
let len = string_arg!(args).chars().count();
return_scalar!(len as f64)
}
@@ -41,9 +41,17 @@ pub fn str_slice(
let end = quantity_arg!(args).unsafe_value().to_f64() as usize;
let input = string_arg!(args);
let output = input.get(start..end).unwrap_or_default();
// Indices refer to Unicode code points, not bytes, so that slicing does not
// depend on the UTF-8 encoding of the input. Out-of-range requests yield an
// empty string, matching the previous byte-based behavior.
let characters = input.chars().collect::<Vec<char>>();
let output = if start <= end && end <= characters.len() {
characters[start..end].iter().collect::<String>()
} else {
String::new()
};
return_string!(borrowed = output)
return_string!(from = &output)
}
pub fn chr(