fix(core/exec): decode shell output via detected encoding

code-rs/Cargo.lock

@@ -720,6 +720,17 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
+[[package]]
+name = "chardetng"
+version = "0.1.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14b8f0b65b7b08ae3c8187e8d77174de20cb6777864c6b832d8ad365999cf1ea"
+dependencies = [
+ "cfg-if",
+ "encoding_rs",
+ "memchr",
+]
+
 [[package]]
 name = "chromiumoxide"
 version = "0.7.0"
@@ -1149,6 +1160,7 @@ dependencies = [
  "async-trait",
  "base64 0.22.1",
  "bytes",
+ "chardetng",
  "chrono",
  "code-app-server-protocol",
  "code-apply-patch",
@@ -1163,6 +1175,7 @@ dependencies = [
  "core-foundation 0.9.4",
  "dirs",
  "dunce",
+ "encoding_rs",
  "env-flags",
  "eventsource-stream",
  "filetime",

code-rs/Cargo.toml

@@ -94,6 +94,7 @@ async-stream = "0.3.6"
 async-trait = "0.1.89"
 base64 = "0.22.1"
 bytes = "1.10.1"
+chardetng = "0.1.17"
 chrono = "0.4.42"
 clap = "4"
 clap_complete = "4"
@@ -108,6 +109,7 @@ dunce = "1.0.5"
 env-flags = "0.1.1"
 env_logger = "0.11.5"
 escargot = "0.5"
+encoding_rs = "0.8.35"
 eventsource-stream = "0.2.3"
 filetime = "0.2"
 futures = "0.3"

code-rs/core/Cargo.toml

@@ -19,6 +19,7 @@ async-trait = { workspace = true }
 base64 = { workspace = true }
 bytes = { workspace = true }
 chrono = { workspace = true, features = ["serde"] }
+chardetng = { workspace = true }
 code-apply-patch = { workspace = true }
 code-file-search = { workspace = true }
 code-mcp-client = { workspace = true }
@@ -32,6 +33,7 @@ agent-client-protocol = "0.4.3"
 dirs = { workspace = true }
 dunce = { workspace = true }
 env-flags = { workspace = true }
+encoding_rs = { workspace = true }
 eventsource-stream = { workspace = true }
 futures = { workspace = true }
 indexmap = { workspace = true }

code-rs/core/src/exec.rs

@@ -21,6 +21,7 @@ use crate::error::CodexErr;
 use crate::error::Result;
 use crate::error::SandboxErr;
 use crate::landlock::spawn_command_under_linux_sandbox;
+use crate::text_encoding::bytes_to_string_smart;
 use crate::protocol::Event;
 use crate::protocol::EventMsg;
 use crate::protocol::OrderMeta;
@@ -281,7 +282,7 @@ impl StreamOutput<String> {
 impl StreamOutput<Vec<u8>> {
     pub fn from_utf8_lossy(&self) -> StreamOutput<String> {
         StreamOutput {
-            text: String::from_utf8_lossy(&self.text).to_string(),
+            text: bytes_to_string_smart(&self.text),
             truncated_after_lines: self.truncated_after_lines,
             truncated_before_bytes: self.truncated_before_bytes,
         }

code-rs/core/src/lib.rs

@@ -100,6 +100,7 @@ pub mod shell;
 pub mod spawn;
 pub mod terminal;
 pub mod otel_init;
+mod text_encoding;
 mod tool_apply_patch;
 mod workflow_validation;
 pub mod turn_diff_tracker;

code-rs/core/src/text_encoding.rs

@@ -0,0 +1,469 @@
+//! Text encoding detection and conversion utilities for shell output.
+//!
+//! Windows users frequently run into code pages such as CP1251 or CP866 when invoking commands
+//! through VS Code. Those bytes show up as invalid UTF-8 and used to be replaced with the standard
+//! Unicode replacement character. We now lean on `chardetng` and `encoding_rs` so we can
+//! automatically detect and decode the vast majority of legacy encodings before falling back to
+//! lossy UTF-8 decoding.
+
+use chardetng::EncodingDetector;
+use encoding_rs::Encoding;
+use encoding_rs::IBM866;
+use encoding_rs::WINDOWS_1252;
+
+/// Attempts to convert arbitrary bytes to UTF-8 with best-effort encoding detection.
+pub fn bytes_to_string_smart(bytes: &[u8]) -> String {
+    if bytes.is_empty() {
+        return String::new();
+    }
+
+    if let Ok(utf8_str) = std::str::from_utf8(bytes) {
+        return utf8_str.to_owned();
+    }
+
+    let encoding = detect_encoding(bytes);
+    decode_bytes(bytes, encoding)
+}
+
+// Windows-1252 reassigns a handful of 0x80-0x9F slots to smart punctuation (curly quotes, dashes,
+// ™). CP866 uses those *same byte values* for uppercase Cyrillic letters. When chardetng sees shell
+// snippets that mix these bytes with ASCII it sometimes guesses IBM866, so “smart quotes” render as
+// Cyrillic garbage (“УФЦ”) in VS Code. However, CP866 uppercase tokens are perfectly valid output
+// (e.g., `ПРИ test`) so we cannot flip every 0x80-0x9F byte to Windows-1252 either. The compromise
+// is to only coerce IBM866 to Windows-1252 when (a) the high bytes are exclusively the punctuation
+// values listed below and (b) we spot adjacent ASCII. This targets the real failure case without
+// clobbering legitimate Cyrillic text. If another code page has a similar collision, introduce a
+// dedicated allowlist (like this one) plus unit tests that capture the actual shell output we want
+// to preserve. Windows-1252 byte values for smart punctuation.
+const WINDOWS_1252_PUNCT_BYTES: [u8; 8] = [
+    0x91, // ‘ (left single quotation mark)
+    0x92, // ’ (right single quotation mark)
+    0x93, // “ (left double quotation mark)
+    0x94, // ” (right double quotation mark)
+    0x95, // • (bullet)
+    0x96, // – (en dash)
+    0x97, // — (em dash)
+    0x99, // ™ (trade mark sign)
+];
+
+fn detect_encoding(bytes: &[u8]) -> &'static Encoding {
+    let mut detector = EncodingDetector::new();
+    detector.feed(bytes, true);
+    let (encoding, _is_confident) = detector.guess_assess(None, true);
+
+    // chardetng occasionally reports IBM866 for short strings that only contain Windows-1252 “smart
+    // punctuation” bytes (0x80-0x9F) because that range maps to Cyrillic letters in IBM866. When
+    // those bytes show up alongside an ASCII word (typical shell output: `"“`test), we know the
+    // intent was likely CP1252 quotes/dashes. Prefer WINDOWS_1252 in that specific situation so we
+    // render the characters users expect instead of Cyrillic junk. References:
+    // - Windows-1252 reserving 0x80-0x9F for curly quotes/dashes:
+    //   https://en.wikipedia.org/wiki/Windows-1252
+    // - CP866 mapping 0x93/0x94/0x96 to Cyrillic letters, so the same bytes show up as “УФЦ” when
+    //   mis-decoded: https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP866.TXT
+    if encoding == IBM866 && looks_like_windows_1252_punctuation(bytes) {
+        return WINDOWS_1252;
+    }
+
+    encoding
+}
+
+fn decode_bytes(bytes: &[u8], encoding: &'static Encoding) -> String {
+    let (decoded, _, had_errors) = encoding.decode(bytes);
+
+    if had_errors {
+        return String::from_utf8_lossy(bytes).into_owned();
+    }
+
+    decoded.into_owned()
+}
+
+/// Detect whether the byte stream looks like Windows-1252 “smart punctuation” wrapped around
+/// otherwise-ASCII text.
+///
+/// Context: IBM866 and Windows-1252 share the 0x80-0x9F slot range. In IBM866 these bytes decode to
+/// Cyrillic letters, whereas Windows-1252 maps them to curly quotes and dashes. chardetng can guess
+/// IBM866 for short snippets that only contain those bytes, which turns shell output such as
+/// `“test”` into unreadable Cyrillic. To avoid that, we treat inputs comprising a handful of bytes
+/// from the problematic range plus ASCII letters as CP1252 punctuation. We deliberately do *not*
+/// cap how many of those punctuation bytes we accept: VS Code frequently prints several quoted
+/// phrases (e.g., `"foo" – "bar"`), and truncating the count would once again mis-decode those as
+/// Cyrillic. If we discover additional encodings with overlapping byte ranges, prefer adding
+/// encoding-specific byte allowlists like `WINDOWS_1252_PUNCT` and tests that exercise real-world
+/// shell snippets.
+fn looks_like_windows_1252_punctuation(bytes: &[u8]) -> bool {
+    let mut saw_extended_punctuation = false;
+    let mut saw_ascii_word = false;
+
+    for &byte in bytes {
+        if byte >= 0xA0 {
+            return false;
+        }
+        if (0x80..=0x9F).contains(&byte) {
+            if !is_windows_1252_punct(byte) {
+                return false;
+            }
+            saw_extended_punctuation = true;
+        }
+        if byte.is_ascii_alphabetic() {
+            saw_ascii_word = true;
+        }
+    }
+
+    saw_extended_punctuation && saw_ascii_word
+}
+
+fn is_windows_1252_punct(byte: u8) -> bool {
+    WINDOWS_1252_PUNCT_BYTES.contains(&byte)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use encoding_rs::BIG5;
+    use encoding_rs::EUC_KR;
+    use encoding_rs::GBK;
+    use encoding_rs::ISO_8859_2;
+    use encoding_rs::ISO_8859_3;
+    use encoding_rs::ISO_8859_4;
+    use encoding_rs::ISO_8859_5;
+    use encoding_rs::ISO_8859_6;
+    use encoding_rs::ISO_8859_7;
+    use encoding_rs::ISO_8859_8;
+    use encoding_rs::ISO_8859_10;
+    use encoding_rs::ISO_8859_13;
+    use encoding_rs::SHIFT_JIS;
+    use encoding_rs::WINDOWS_874;
+    use encoding_rs::WINDOWS_1250;
+    use encoding_rs::WINDOWS_1251;
+    use encoding_rs::WINDOWS_1253;
+    use encoding_rs::WINDOWS_1254;
+    use encoding_rs::WINDOWS_1255;
+    use encoding_rs::WINDOWS_1256;
+    use encoding_rs::WINDOWS_1257;
+    use encoding_rs::WINDOWS_1258;
+    use pretty_assertions::assert_eq;
+
+    #[test]
+    fn test_utf8_passthrough() {
+        // Fast path: when UTF-8 is valid we should avoid copies and return as-is.
+        let utf8_text = "Hello, мир! 世界";
+        let bytes = utf8_text.as_bytes();
+        assert_eq!(bytes_to_string_smart(bytes), utf8_text);
+    }
+
+    #[test]
+    fn test_cp1251_russian_text() {
+        // Cyrillic text emitted by PowerShell/WSL in CP1251 should decode cleanly.
+        let bytes = b"\xEF\xF0\xE8\xEC\xE5\xF0"; // "пример" encoded with Windows-1251
+        assert_eq!(bytes_to_string_smart(bytes), "пример");
+    }
+
+    #[test]
+    fn test_cp1251_privet_word() {
+        // Regression: CP1251 words like "Привет" must not be mis-identified as Windows-1252.
+        let bytes = b"\xCF\xF0\xE8\xE2\xE5\xF2"; // "Привет" encoded with Windows-1251
+        assert_eq!(bytes_to_string_smart(bytes), "Привет");
+    }
+
+    #[test]
+    fn test_koi8_r_privet_word() {
+        // KOI8-R output should decode to the original Cyrillic as well.
+        let bytes = b"\xF0\xD2\xC9\xD7\xC5\xD4"; // "Привет" encoded with KOI8-R
+        assert_eq!(bytes_to_string_smart(bytes), "Привет");
+    }
+
+    #[test]
+    fn test_cp866_russian_text() {
+        // Legacy consoles (cmd.exe) commonly emit CP866 bytes for Cyrillic content.
+        let bytes = b"\xAF\xE0\xA8\xAC\xA5\xE0"; // "пример" encoded with CP866
+        assert_eq!(bytes_to_string_smart(bytes), "пример");
+    }
+
+    #[test]
+    fn test_cp866_uppercase_text() {
+        // Ensure the IBM866 heuristic still returns IBM866 for uppercase-only words.
+        let bytes = b"\x8F\x90\x88"; // "ПРИ" encoded with CP866 uppercase letters
+        assert_eq!(bytes_to_string_smart(bytes), "ПРИ");
+    }
+
+    #[test]
+    fn test_cp866_uppercase_followed_by_ascii() {
+        // Regression test: uppercase CP866 tokens next to ASCII text should not be treated as
+        // CP1252.
+        let bytes = b"\x8F\x90\x88 test"; // "ПРИ test" encoded with CP866 uppercase letters followed by ASCII
+        assert_eq!(bytes_to_string_smart(bytes), "ПРИ test");
+    }
+
+    #[test]
+    fn test_windows_1252_quotes() {
+        // Smart detection should map Windows-1252 punctuation into proper Unicode.
+        let bytes = b"\x93\x94test";
+        assert_eq!(bytes_to_string_smart(bytes), "\u{201C}\u{201D}test");
+    }
+
+    #[test]
+    fn test_windows_1252_multiple_quotes() {
+        // Longer snippets of punctuation (e.g., “foo” – “bar”) should still flip to CP1252.
+        let bytes = b"\x93foo\x94 \x96 \x93bar\x94";
+        assert_eq!(
+            bytes_to_string_smart(bytes),
+            "\u{201C}foo\u{201D} \u{2013} \u{201C}bar\u{201D}"
+        );
+    }
+
+    #[test]
+    fn test_windows_1252_privet_gibberish_is_preserved() {
+        // Windows-1252 cannot encode Cyrillic; if the input literally contains "ПÑ..." we should not "fix" it.
+        let bytes = "Привет".as_bytes();
+        assert_eq!(bytes_to_string_smart(bytes), "Привет");
+    }
+
+    #[test]
+    fn test_iso8859_1_latin_text() {
+        // ISO-8859-1 (code page 28591) is the Latin segment used by LatArCyrHeb.
+        // encoding_rs unifies ISO-8859-1 with Windows-1252, so reuse that constant here.
+        let (encoded, _, had_errors) = WINDOWS_1252.encode("Hello");
+        assert!(!had_errors, "failed to encode Latin sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Hello");
+    }
+
+    #[test]
+    fn test_iso8859_2_central_european_text() {
+        // ISO-8859-2 (code page 28592) covers additional Central European glyphs.
+        let (encoded, _, had_errors) = ISO_8859_2.encode("Příliš žluťoučký kůň");
+        assert!(!had_errors, "failed to encode ISO-8859-2 sample");
+        assert_eq!(
+            bytes_to_string_smart(encoded.as_ref()),
+            "Příliš žluťoučký kůň"
+        );
+    }
+
+    #[test]
+    fn test_iso8859_3_southern_european_text() {
+        // ISO-8859-3 (code page 28593) covers Southern European languages.
+        let sample = "Ħelow";
+        let (encoded, _, had_errors) = ISO_8859_3.encode(sample);
+        assert!(!had_errors, "failed to encode ISO-8859-3 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_iso8859_4_northern_european_text() {
+        // ISO-8859-4 (code page 28594) covers Northern European languages.
+        let sample = "Pērkons";
+        let (encoded, _, had_errors) = ISO_8859_4.encode(sample);
+        assert!(!had_errors, "failed to encode ISO-8859-4 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_iso8859_5_cyrillic_text() {
+        // ISO-8859-5 (code page 28595) is another Cyrillic encoding.
+        let sample = "Привет";
+        let (encoded, _, had_errors) = ISO_8859_5.encode(sample);
+        assert!(!had_errors, "failed to encode ISO-8859-5 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_iso8859_6_arabic_text() {
+        // ISO-8859-6 (code page 28596) covers Arabic characters.
+        let sample = "مرحبا";
+        let (encoded, _, had_errors) = ISO_8859_6.encode(sample);
+        assert!(!had_errors, "failed to encode ISO-8859-6 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_iso8859_7_greek_text() {
+        // ISO-8859-7 (code page 28597) covers Greek characters.
+        let sample = "Καλημέρα";
+        let (encoded, _, had_errors) = ISO_8859_7.encode(sample);
+        assert!(!had_errors, "failed to encode ISO-8859-7 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_iso8859_8_hebrew_text() {
+        // ISO-8859-8 (code page 28598) covers Hebrew characters.
+        let sample = "שלום";
+        let (encoded, _, had_errors) = ISO_8859_8.encode(sample);
+        assert!(!had_errors, "failed to encode ISO-8859-8 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_iso8859_10_nordic_text() {
+        // ISO-8859-10 (code page 28600) covers Nordic languages.
+        let sample = "Ísland";
+        let (encoded, _, had_errors) = ISO_8859_10.encode(sample);
+        assert!(!had_errors, "failed to encode ISO-8859-10 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_iso8859_13_baltic_text() {
+        // ISO-8859-13 (code page 28603) covers Baltic languages.
+        let sample = "Sveiki";
+        let (encoded, _, had_errors) = ISO_8859_13.encode(sample);
+        assert!(!had_errors, "failed to encode ISO-8859-13 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_koi8_russian_text() {
+        // KOI8-R encoding for Russian text.
+        let sample = "Привет";
+        let (encoded, _, had_errors) = encoding_rs::KOI8_R.encode(sample);
+        assert!(!had_errors, "failed to encode KOI8-R sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_koi8_u_ukrainian_text() {
+        // KOI8-U encoding for Ukrainian text.
+        let sample = "Привіт";
+        let (encoded, _, had_errors) = encoding_rs::KOI8_U.encode(sample);
+        assert!(!had_errors, "failed to encode KOI8-U sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_windows_874_thai_text() {
+        // Windows-874 encoding for Thai text.
+        let sample = "สวัสดี";
+        let (encoded, _, had_errors) = WINDOWS_874.encode(sample);
+        assert!(!had_errors, "failed to encode Windows-874 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_windows_1250_central_european_text() {
+        // Windows-1250 encoding for Central European languages.
+        let sample = "Dobrý den";
+        let (encoded, _, had_errors) = WINDOWS_1250.encode(sample);
+        assert!(!had_errors, "failed to encode Windows-1250 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_windows_1251_cyrillic_text() {
+        // Windows-1251 encoding for Cyrillic text.
+        let sample = "Привет";
+        let (encoded, _, had_errors) = WINDOWS_1251.encode(sample);
+        assert!(!had_errors, "failed to encode Windows-1251 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_windows_1253_greek_text() {
+        // Windows-1253 encoding for Greek text.
+        let sample = "Γειά σου";
+        let (encoded, _, had_errors) = WINDOWS_1253.encode(sample);
+        assert!(!had_errors, "failed to encode Windows-1253 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_windows_1254_turkish_text() {
+        // Windows-1254 encoding for Turkish text.
+        let sample = "İstanbul";
+        let (encoded, _, had_errors) = WINDOWS_1254.encode(sample);
+        assert!(!had_errors, "failed to encode Windows-1254 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_windows_1255_hebrew_text() {
+        // Windows-1255 encoding for Hebrew text.
+        let sample = "שלום";
+        let (encoded, _, had_errors) = WINDOWS_1255.encode(sample);
+        assert!(!had_errors, "failed to encode Windows-1255 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_windows_1256_arabic_text() {
+        // Windows-1256 encoding for Arabic text.
+        let sample = "مرحبا";
+        let (encoded, _, had_errors) = WINDOWS_1256.encode(sample);
+        assert!(!had_errors, "failed to encode Windows-1256 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_windows_1257_baltic_text() {
+        // Windows-1257 encoding for Baltic languages.
+        let sample = "Pērkons";
+        let (encoded, _, had_errors) = WINDOWS_1257.encode(sample);
+        assert!(!had_errors, "failed to encode Windows-1257 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_windows_1258_vietnamese_text() {
+        // Windows-1258 encoding for Vietnamese text.
+        let sample = "Xin chào";
+        let (encoded, _, had_errors) = WINDOWS_1258.encode(sample);
+        assert!(!had_errors, "failed to encode Windows-1258 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_big5_traditional_chinese_text() {
+        // Big5 encoding for Traditional Chinese text.
+        let sample = "繁體";
+        let (encoded, _, had_errors) = BIG5.encode(sample);
+        assert!(!had_errors, "failed to encode Big5 sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_shift_jis_japanese_text() {
+        // Shift_JIS encoding for Japanese text.
+        let sample = "こんにちは";
+        let (encoded, _, had_errors) = SHIFT_JIS.encode(sample);
+        assert!(!had_errors, "failed to encode Shift_JIS sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_euc_kr_korean_text() {
+        // EUC-KR encoding for Korean text.
+        let sample = "안녕하세요";
+        let (encoded, _, had_errors) = EUC_KR.encode(sample);
+        assert!(!had_errors, "failed to encode EUC-KR sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_gbk_simplified_chinese_text() {
+        // GBK encoding for Simplified Chinese text.
+        let sample = "简体";
+        let (encoded, _, had_errors) = GBK.encode(sample);
+        assert!(!had_errors, "failed to encode GBK sample");
+        assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample);
+    }
+
+    #[test]
+    fn test_utf8_latin1_mixed_text() {
+        // Mixed ASCII and Latin-1 characters should remain intact.
+        let bytes = b"caf\xe9";
+        assert_eq!(bytes_to_string_smart(bytes), "caf\u{e9}");
+    }
+
+    #[test]
+    fn test_escape_sequences_preserved() {
+        // ANSI escape sequences should remain intact.
+        let bytes = b"\x1b[31mred\x1b[0m";
+        assert_eq!(bytes_to_string_smart(bytes), "\x1b[31mred\x1b[0m");
+    }
+
+    #[test]
+    fn test_invalid_utf8_fallback() {
+        // Invalid UTF-8 should fall back to lossy decoding.
+        let invalid_bytes = vec![0x80, 0x81, 0x82];
+        let result = bytes_to_string_smart(&invalid_bytes);
+        assert_eq!(result, String::from_utf8_lossy(&invalid_bytes));
+    }
+}