fix panic on unicode string truncation

[atakurt]

thread 'main' panicked at 'assertion failed: self.is_char_boundary(new_len)', /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/alloc/src/string.rs:1192:13
stack backtrace:
0: 0x105c19654 - std::backtrace_rs::backtrace::libunwind::trace::hd9b9204470182ffc
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/../../backtrace/src/backtrace/libunwind.rs:90:5
1: 0x105c19654 - std::backtrace_rs::backtrace::trace_unsynchronized::hdc48b9180be04761
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
2: 0x105c19654 - std::sys_common::backtrace::_print_fmt::h8824de041faf4112
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/sys_common/backtrace.rs:67:5
3: 0x105c19654 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h8d168763f08a78b9
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/sys_common/backtrace.rs:46:22
4: 0x105c38a6c - core::fmt::write::h0fa46d4891ed63f1
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/fmt/mod.rs:1115:17
5: 0x105c1679a - std::io::Write::write_fmt::hacbf9e10e17149ce
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/io/mod.rs:1665:15
6: 0x105c1b30f - std::sys_common::backtrace::_print::h26213154a400cfbe
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/sys_common/backtrace.rs:49:5
7: 0x105c1b30f - std::sys_common::backtrace::print::h0f4ab60f53d1f93b
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/sys_common/backtrace.rs:36:9
8: 0x105c1b30f - std::panicking::default_hook::{{closure}}::hee7cf38dedfa76fc
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/panicking.rs:208:50
9: 0x105c1ae0d - std::panicking::default_hook::hd12252412832a45f
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/panicking.rs:225:9
10: 0x105c1ba10 - std::panicking::rust_panic_with_hook::h521fd09e07057893
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/panicking.rs:622:17
11: 0x105c1b489 - std::panicking::begin_panic_handler::{{closure}}::h3588a5a9cca9e8e5
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/panicking.rs:517:13
12: 0x105c19af8 - std::sys_common::backtrace::__rust_end_short_backtrace::h86522c151a195ff9
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/sys_common/backtrace.rs:141:18
13: 0x105c1b41a - rust_begin_unwind
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/panicking.rs:515:5
14: 0x105c4425f - core::panicking::panic_fmt::h8ab5bf5f27d0a26a
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/panicking.rs:92:14
15: 0x105c441b7 - core::panicking::panic::h2c89bafc52ad2c2f
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/panicking.rs:50:5
16: 0x105940d35 - alloc::string::String::truncate::hfd1dbc74aa665454
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/alloc/src/string.rs:1192:13
17: 0x10593cde7 - tidy_viewer::datatype::trunc_strings::{{closure}}::heb992dbf1e85ba85
at /Users/ismaila.kurt/Documents/github/tv/src/datatype.rs:111:17
18: 0x105944c67 - core::iter::adapters::map::map_fold::{{closure}}::hf06b84d34dd2ba4d
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/iter/adapters/map.rs:82:28
19: 0x105944777 - core::iter::adapters::map::map_fold::{{closure}}::h5b0d02eefe5719f2
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/iter/adapters/map.rs:82:21
20: 0x105944a21 - core::iter::adapters::map::map_fold::{{closure}}::hc1b95e801fac36eb
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/iter/adapters/map.rs:82:21
21: 0x1059608b1 - core::iter::traits::iterator::Iterator::fold::hf37edb4b917d8fe4
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/iter/traits/iterator.rs:2174:21
22: 0x10594371a - <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::fold::hc47d0197acf64739
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/iter/adapters/map.rs:122:9
23: 0x10594363a - <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::fold::hbb8e03c1e832fed1
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/iter/adapters/map.rs:122:9
24: 0x10594349c - <core::iter::adapters::map::Map<I,F> as core::iter::traits::iterator::Iterator>::fold::h9d17182cacd1902a
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/iter/adapters/map.rs:122:9
25: 0x1059441b6 - core::iter::traits::iterator::Iterator::for_each::h807ae5e76521d65d
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/iter/traits/iterator.rs:737:9
26: 0x1059582f2 - <alloc::vec::Vec<T,A> as alloc::vec::spec_extend::SpecExtend<T,I>>::spec_extend::h453b98c068965e90
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/alloc/src/vec/spec_extend.rs:40:17
27: 0x105954b00 - <alloc::vec::Vec as alloc::vec::spec_from_iter_nested::SpecFromIterNested<T,I>>::from_iter::h1dbf1661935fbcb1
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/alloc/src/vec/spec_from_iter_nested.rs:56:9
28: 0x1059587d7 - <alloc::vec::Vec as alloc::vec::spec_from_iter::SpecFromIter<T,I>>::from_iter::habc4a915c81ecf03
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/alloc/src/vec/spec_from_iter.rs:33:9
29: 0x105957f29 - <alloc::vec::Vec as core::iter::traits::collect::FromIterator>::from_iter::hf1798c3c9048603b
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/alloc/src/vec/mod.rs:2453:9
30: 0x105943f47 - core::iter::traits::iterator::Iterator::collect::hb5b39395b406366f
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/iter/traits/iterator.rs:1749:9
31: 0x105958c91 - tidy_viewer::datatype::trunc_strings::ha96eb77958ed6023
at /Users/ismaila.kurt/Documents/github/tv/src/datatype.rs:103:5
32: 0x10595cf39 - tidy_viewer::main::h3b1a4f5a638a679f
at /Users/ismaila.kurt/Documents/github/tv/src/main.rs:238:17
33: 0x1059504ce - core::ops::function::FnOnce::call_once::hafdcbf59c695878a
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/ops/function.rs:227:5
34: 0x105961d01 - std::sys_common::backtrace::__rust_begin_short_backtrace::h6969d047d0815eec
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/sys_common/backtrace.rs:125:18
35: 0x10594df14 - std::rt::lang_start::{{closure}}::h4402b9e959ba939d
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/rt.rs:63:18
36: 0x105c1be59 - core::ops::function::impls::<impl core::ops::function::FnOnce for &F>::call_once::h5adc1669b25f044b
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/core/src/ops/function.rs:259:13
37: 0x105c1be59 - std::panicking::try::do_call::h405f88a3fec16587
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/panicking.rs:401:40
38: 0x105c1be59 - std::panicking::try::h86d9af92fb3c6022
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/panicking.rs:365:19
39: 0x105c1be59 - std::panic::catch_unwind::h67150880d2b87901
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/panic.rs:434:14
40: 0x105c1be59 - std::rt::lang_start_internal::{{closure}}::h548e43d03023c928
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/rt.rs:45:48
41: 0x105c1be59 - std::panicking::try::do_call::h1383affbfec23244
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/panicking.rs:401:40
42: 0x105c1be59 - std::panicking::try::hf51149bda74e8587
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/panicking.rs:365:19
43: 0x105c1be59 - std::panic::catch_unwind::h8280208a4f5e7cf4
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/panic.rs:434:14
44: 0x105c1be59 - std::rt::lang_start_internal::h5f9b030f0a63c040
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/rt.rs:45:20
45: 0x10594dede - std::rt::lang_start::hd0de7346a93394b9
at /rustc/c8dfcfe046a7680554bf4eb612bad840e7631c4b/library/std/src/rt.rs:62:5
46: 0x1059605a6 - _main

[alexhallam]

Hi @atakurt. Thanks for the pull request. Could you provide an example of a csv this fails on? It would be a nice test case.

[atakurt]

Hi @alexhallam i have added the example csv, you can reproduce the unicode truncation.

test.csv

[alexhallam]

Thanks! I will checkout that file. There are two things I want to make sure are taken care of before I merge.

• Add test.csv to data directory and possibly give it a more descriptive name. https://github.com/alexhallam/tv/tree/main/data
• Write a test at the bottom of this file https://github.com/alexhallam/tv/blob/main/src/datatype/sigfig.rs#L542

[alexhallam]

I got a second to look at this. I just want to verify. This is the contents of the file. I noticed there was not a comma between aColumn and bColumn so I added one:

aColumn,bColumn,cColumn,dColumn,eColumn,fColumn,gColumn
1,üÜğĞçÇşŞöÖ,üÜğĞçÇşŞöÖ üÜğĞçÇşŞöÖ,77,TR,77,77

With your code I get the following:

cat test.csv | tv

      tv dim: 1 x 7
      aColumn bColumn    cColumn              dColumn eColumn fColumn gColumn 
1     1       üÜğĞçÇşŞöÖ üÜğĞçÇşŞöÖ üÜğĞçÇşŞ …77      TR      77      77      

I think this is very nice. Thanks for the contribution, I will come back around and add a test.

[atakurt]

@alexhallam how about adding test like this in main.rs ?

#[test]
fn test_unicode_truncation() {
    assert_eq!(
        datatype::trunc_strings(&["üÜğĞçÇşŞöÖ üÜğĞçÇşŞöÖ"], 14),
        vec!["üÜğĞçÇşŞöÖ üÜ …"]
    );
}

if it's ok i can pr that.

[alexhallam]

I like that. It is much better than nothing.