diff --git a/Cargo.lock b/Cargo.lock index dfb866147d4..78eb9b3694f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -679,7 +679,7 @@ dependencies = [ [[package]] name = "diplomat" version = "0.7.0" -source = "git+https://github.com/rust-diplomat/diplomat.git?rev=bf3ebcbdfbd3ce5da2086801b47b54487b1b727c#bf3ebcbdfbd3ce5da2086801b47b54487b1b727c" +source = "git+https://github.com/rust-diplomat/diplomat.git?rev=3167e39e89ae612233706f1ade3d9e8a5c6ad358#3167e39e89ae612233706f1ade3d9e8a5c6ad358" dependencies = [ "diplomat_core", "proc-macro2", @@ -697,7 +697,7 @@ dependencies = [ [[package]] name = "diplomat-runtime" version = "0.7.0" -source = "git+https://github.com/rust-diplomat/diplomat.git?rev=bf3ebcbdfbd3ce5da2086801b47b54487b1b727c#bf3ebcbdfbd3ce5da2086801b47b54487b1b727c" +source = "git+https://github.com/rust-diplomat/diplomat.git?rev=3167e39e89ae612233706f1ade3d9e8a5c6ad358#3167e39e89ae612233706f1ade3d9e8a5c6ad358" dependencies = [ "log", ] @@ -705,7 +705,7 @@ dependencies = [ [[package]] name = "diplomat-tool" version = "0.7.0" -source = "git+https://github.com/rust-diplomat/diplomat.git?rev=bf3ebcbdfbd3ce5da2086801b47b54487b1b727c#bf3ebcbdfbd3ce5da2086801b47b54487b1b727c" +source = "git+https://github.com/rust-diplomat/diplomat.git?rev=3167e39e89ae612233706f1ade3d9e8a5c6ad358#3167e39e89ae612233706f1ade3d9e8a5c6ad358" dependencies = [ "askama", "clap", @@ -725,7 +725,7 @@ dependencies = [ [[package]] name = "diplomat_core" version = "0.7.0" -source = "git+https://github.com/rust-diplomat/diplomat.git?rev=bf3ebcbdfbd3ce5da2086801b47b54487b1b727c#bf3ebcbdfbd3ce5da2086801b47b54487b1b727c" +source = "git+https://github.com/rust-diplomat/diplomat.git?rev=3167e39e89ae612233706f1ade3d9e8a5c6ad358#3167e39e89ae612233706f1ade3d9e8a5c6ad358" dependencies = [ "displaydoc", "either", diff --git a/Cargo.toml b/Cargo.toml index 409db1922e0..39e9bcb24e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -209,10 +209,10 @@ icu_benchmark_macros = { path = "tools/benchmark/macros" } # The version here can either be a `version = ".."` spec or `git = "https://github.com/rust-diplomat/diplomat", rev = ".."` # Diplomat must be published preceding a new ICU4X release but may use git versions in between -diplomat = { git = "https://github.com/rust-diplomat/diplomat.git", rev = "bf3ebcbdfbd3ce5da2086801b47b54487b1b727c" } -diplomat-runtime = { git = "https://github.com/rust-diplomat/diplomat.git", rev = "bf3ebcbdfbd3ce5da2086801b47b54487b1b727c" } -diplomat_core = { git = "https://github.com/rust-diplomat/diplomat.git", rev = "bf3ebcbdfbd3ce5da2086801b47b54487b1b727c" } -diplomat-tool = { git = "https://github.com/rust-diplomat/diplomat.git", rev = "bf3ebcbdfbd3ce5da2086801b47b54487b1b727c" } +diplomat = { git = "https://github.com/rust-diplomat/diplomat.git", rev = "3167e39e89ae612233706f1ade3d9e8a5c6ad358" } +diplomat-runtime = { git = "https://github.com/rust-diplomat/diplomat.git", rev = "3167e39e89ae612233706f1ade3d9e8a5c6ad358" } +diplomat_core = { git = "https://github.com/rust-diplomat/diplomat.git", rev = "3167e39e89ae612233706f1ade3d9e8a5c6ad358" } +diplomat-tool = { git = "https://github.com/rust-diplomat/diplomat.git", rev = "3167e39e89ae612233706f1ade3d9e8a5c6ad358" } # Size optimized builds [profile.release-opt-size] diff --git a/ffi/capi/bindings/c/ICU4XLocaleFallbackIterator.h b/ffi/capi/bindings/c/ICU4XLocaleFallbackIterator.h index 11260be1453..e2dd7a1a3ba 100644 --- a/ffi/capi/bindings/c/ICU4XLocaleFallbackIterator.h +++ b/ffi/capi/bindings/c/ICU4XLocaleFallbackIterator.h @@ -23,8 +23,6 @@ extern "C" { ICU4XLocale* ICU4XLocaleFallbackIterator_get(const ICU4XLocaleFallbackIterator* self); void ICU4XLocaleFallbackIterator_step(ICU4XLocaleFallbackIterator* self); - -ICU4XLocale* ICU4XLocaleFallbackIterator_next(ICU4XLocaleFallbackIterator* self); void ICU4XLocaleFallbackIterator_destroy(ICU4XLocaleFallbackIterator* self); #ifdef __cplusplus diff --git a/ffi/capi/bindings/cpp/ICU4XLocaleFallbackIterator.h b/ffi/capi/bindings/cpp/ICU4XLocaleFallbackIterator.h index 11260be1453..e2dd7a1a3ba 100644 --- a/ffi/capi/bindings/cpp/ICU4XLocaleFallbackIterator.h +++ b/ffi/capi/bindings/cpp/ICU4XLocaleFallbackIterator.h @@ -23,8 +23,6 @@ extern "C" { ICU4XLocale* ICU4XLocaleFallbackIterator_get(const ICU4XLocaleFallbackIterator* self); void ICU4XLocaleFallbackIterator_step(ICU4XLocaleFallbackIterator* self); - -ICU4XLocale* ICU4XLocaleFallbackIterator_next(ICU4XLocaleFallbackIterator* self); void ICU4XLocaleFallbackIterator_destroy(ICU4XLocaleFallbackIterator* self); #ifdef __cplusplus diff --git a/ffi/capi/bindings/cpp/ICU4XLocaleFallbackIterator.hpp b/ffi/capi/bindings/cpp/ICU4XLocaleFallbackIterator.hpp index c41b5af38d4..93c47eb01e8 100644 --- a/ffi/capi/bindings/cpp/ICU4XLocaleFallbackIterator.hpp +++ b/ffi/capi/bindings/cpp/ICU4XLocaleFallbackIterator.hpp @@ -43,12 +43,6 @@ class ICU4XLocaleFallbackIterator { * See the [Rust documentation for `step`](https://docs.rs/icu/latest/icu/locid_transform/fallback/struct.LocaleFallbackIterator.html#method.step) for more information. */ void step(); - - /** - * A combination of `get` and `step`. Returns the value that `get` would return - * and advances the iterator until hitting `und`. - */ - std::optional next(); inline const capi::ICU4XLocaleFallbackIterator* AsFFI() const { return this->inner.get(); } inline capi::ICU4XLocaleFallbackIterator* AsFFIMut() { return this->inner.get(); } inline explicit ICU4XLocaleFallbackIterator(capi::ICU4XLocaleFallbackIterator* i) : inner(i) {} @@ -67,14 +61,4 @@ inline ICU4XLocale ICU4XLocaleFallbackIterator::get() const { inline void ICU4XLocaleFallbackIterator::step() { capi::ICU4XLocaleFallbackIterator_step(this->inner.get()); } -inline std::optional ICU4XLocaleFallbackIterator::next() { - auto diplomat_optional_raw_out_value = capi::ICU4XLocaleFallbackIterator_next(this->inner.get()); - std::optional diplomat_optional_out_value; - if (diplomat_optional_raw_out_value != nullptr) { - diplomat_optional_out_value = ICU4XLocale(diplomat_optional_raw_out_value); - } else { - diplomat_optional_out_value = std::nullopt; - } - return diplomat_optional_out_value; -} #endif diff --git a/ffi/capi/bindings/dart/Collator.g.dart b/ffi/capi/bindings/dart/Collator.g.dart index 39b7265172b..14738386f22 100644 --- a/ffi/capi/bindings/dart/Collator.g.dart +++ b/ffi/capi/bindings/dart/Collator.g.dart @@ -43,13 +43,13 @@ final class Collator implements ffi.Finalizable { /// to the WHATWG Encoding Standard. /// /// See the [Rust documentation for `compare_utf16`](https://docs.rs/icu/latest/icu/collator/struct.Collator.html#method.compare_utf16) for more information. - Ordering compare(String left, String right) { + int compare(String left, String right) { final temp = ffi2.Arena(); final leftView = left.utf16View; final rightView = right.utf16View; - final result = _ICU4XCollator_compare_utf16(_ffi, leftView.allocIn(temp), leftView.length, rightView.allocIn(temp), rightView.length); + final result = _ICU4XCollator_compare_utf16_(_ffi, leftView.allocIn(temp), leftView.length, rightView.allocIn(temp), rightView.length); temp.releaseAll(); - return Ordering.values.firstWhere((v) => v._ffi == result); + return result; } /// The resolved options showing how the default options, the requested options, @@ -73,10 +73,10 @@ external void _ICU4XCollator_destroy(ffi.Pointer self); // ignore: non_constant_identifier_names external _ResultOpaqueInt32 _ICU4XCollator_create_v1(ffi.Pointer provider, ffi.Pointer locale, _CollatorOptionsFfi options); -@meta.ResourceIdentifier('ICU4XCollator_compare_utf16') -@ffi.Native, ffi.Pointer, ffi.Size, ffi.Pointer, ffi.Size)>(isLeaf: true, symbol: 'ICU4XCollator_compare_utf16') +@meta.ResourceIdentifier('ICU4XCollator_compare_utf16_') +@ffi.Native, ffi.Pointer, ffi.Size, ffi.Pointer, ffi.Size)>(isLeaf: true, symbol: 'ICU4XCollator_compare_utf16_') // ignore: non_constant_identifier_names -external int _ICU4XCollator_compare_utf16(ffi.Pointer self, ffi.Pointer leftData, int leftLength, ffi.Pointer rightData, int rightLength); +external int _ICU4XCollator_compare_utf16_(ffi.Pointer self, ffi.Pointer leftData, int leftLength, ffi.Pointer rightData, int rightLength); @meta.ResourceIdentifier('ICU4XCollator_resolved_options') @ffi.Native<_ResolvedCollatorOptionsFfi Function(ffi.Pointer)>(isLeaf: true, symbol: 'ICU4XCollator_resolved_options') diff --git a/ffi/capi/bindings/dart/List.g.dart b/ffi/capi/bindings/dart/List.g.dart deleted file mode 100644 index 052592ba4c2..00000000000 --- a/ffi/capi/bindings/dart/List.g.dart +++ /dev/null @@ -1,90 +0,0 @@ -// generated by diplomat-tool - -part of 'lib.g.dart'; - -/// A list of strings -final class List implements ffi.Finalizable { - final ffi.Pointer _ffi; - - // These are "used" in the sense that they keep dependencies alive - // ignore: unused_field - final core.List _selfEdge; - - // This takes in a list of lifetime edges (including for &self borrows) - // corresponding to data this may borrow from. These should be flat arrays containing - // references to objects, and this object will hold on to them to keep them alive and - // maintain borrow validity. - List._fromFfi(this._ffi, this._selfEdge) { - if (_selfEdge.isEmpty) { - _finalizer.attach(this, _ffi.cast()); - } - } - - static final _finalizer = ffi.NativeFinalizer(ffi.Native.addressOf(_ICU4XList_destroy)); - - /// Create a new list of strings - factory List() { - final result = _ICU4XList_create(); - return List._fromFfi(result, []); - } - - /// Create a new list of strings with preallocated space to hold - /// at least `capacity` elements - factory List.withCapacity(int capacity) { - final result = _ICU4XList_create_with_capacity(capacity); - return List._fromFfi(result, []); - } - - /// Push a string to the list - /// - /// Ill-formed input is treated as if errors had been replaced with REPLACEMENT CHARACTERs according - /// to the WHATWG Encoding Standard. - void push(String val) { - final temp = ffi2.Arena(); - final valView = val.utf8View; - _ICU4XList_push(_ffi, valView.allocIn(temp), valView.length); - temp.releaseAll(); - } - - /// The number of elements in this list - int get length { - final result = _ICU4XList_len(_ffi); - return result; - } - - /// Whether this list is empty - bool get isEmpty { - final result = _ICU4XList_is_empty(_ffi); - return result; - } -} - -@meta.ResourceIdentifier('ICU4XList_destroy') -@ffi.Native)>(isLeaf: true, symbol: 'ICU4XList_destroy') -// ignore: non_constant_identifier_names -external void _ICU4XList_destroy(ffi.Pointer self); - -@meta.ResourceIdentifier('ICU4XList_create') -@ffi.Native Function()>(isLeaf: true, symbol: 'ICU4XList_create') -// ignore: non_constant_identifier_names -external ffi.Pointer _ICU4XList_create(); - -@meta.ResourceIdentifier('ICU4XList_create_with_capacity') -@ffi.Native Function(ffi.Size)>(isLeaf: true, symbol: 'ICU4XList_create_with_capacity') -// ignore: non_constant_identifier_names -external ffi.Pointer _ICU4XList_create_with_capacity(int capacity); - -@meta.ResourceIdentifier('ICU4XList_push') -@ffi.Native, ffi.Pointer, ffi.Size)>(isLeaf: true, symbol: 'ICU4XList_push') -// ignore: non_constant_identifier_names -external void _ICU4XList_push(ffi.Pointer self, ffi.Pointer valData, int valLength); - -@meta.ResourceIdentifier('ICU4XList_len') -@ffi.Native)>(isLeaf: true, symbol: 'ICU4XList_len') -// ignore: non_constant_identifier_names -external int _ICU4XList_len(ffi.Pointer self); - -@meta.ResourceIdentifier('ICU4XList_is_empty') -@ffi.Native)>(isLeaf: true, symbol: 'ICU4XList_is_empty') -// ignore: non_constant_identifier_names -external bool _ICU4XList_is_empty(ffi.Pointer self); diff --git a/ffi/capi/bindings/dart/ListFormatter.g.dart b/ffi/capi/bindings/dart/ListFormatter.g.dart index 39057958844..3d0ea8f2ef0 100644 --- a/ffi/capi/bindings/dart/ListFormatter.g.dart +++ b/ffi/capi/bindings/dart/ListFormatter.g.dart @@ -64,9 +64,12 @@ final class ListFormatter implements ffi.Finalizable { /// See the [Rust documentation for `format`](https://docs.rs/icu/latest/icu/list/struct.ListFormatter.html#method.format) for more information. /// /// Throws [Error] on failure. - String format(List list) { + String format(core.List list) { + final temp = ffi2.Arena(); + final listView = list.utf16View; final writeable = _Writeable(); - final result = _ICU4XListFormatter_format(_ffi, list._ffi, writeable._ffi); + final result = _ICU4XListFormatter_format_utf16(_ffi, listView.allocIn(temp), listView.length, writeable._ffi); + temp.releaseAll(); if (!result.isOk) { throw Error.values.firstWhere((v) => v._ffi == result.union.err); } @@ -94,7 +97,7 @@ external _ResultOpaqueInt32 _ICU4XListFormatter_create_or_with_length(ffi.Pointe // ignore: non_constant_identifier_names external _ResultOpaqueInt32 _ICU4XListFormatter_create_unit_with_length(ffi.Pointer provider, ffi.Pointer locale, int length); -@meta.ResourceIdentifier('ICU4XListFormatter_format') -@ffi.Native<_ResultVoidInt32 Function(ffi.Pointer, ffi.Pointer, ffi.Pointer)>(isLeaf: true, symbol: 'ICU4XListFormatter_format') +@meta.ResourceIdentifier('ICU4XListFormatter_format_utf16') +@ffi.Native<_ResultVoidInt32 Function(ffi.Pointer, ffi.Pointer<_SliceUtf16>, ffi.Size, ffi.Pointer)>(isLeaf: true, symbol: 'ICU4XListFormatter_format_utf16') // ignore: non_constant_identifier_names -external _ResultVoidInt32 _ICU4XListFormatter_format(ffi.Pointer self, ffi.Pointer list, ffi.Pointer writeable); +external _ResultVoidInt32 _ICU4XListFormatter_format_utf16(ffi.Pointer self, ffi.Pointer<_SliceUtf16> listData, int listLength, ffi.Pointer writeable); diff --git a/ffi/capi/bindings/dart/Locale.g.dart b/ffi/capi/bindings/dart/Locale.g.dart index 738e7db2e7a..e430987a04c 100644 --- a/ffi/capi/bindings/dart/Locale.g.dart +++ b/ffi/capi/bindings/dart/Locale.g.dart @@ -5,7 +5,7 @@ part of 'lib.g.dart'; /// An ICU4X Locale, capable of representing strings like `"en-US"`. /// /// See the [Rust documentation for `Locale`](https://docs.rs/icu/latest/icu/locid/struct.Locale.html) for more information. -final class Locale implements ffi.Finalizable { +final class Locale implements ffi.Finalizable, core.Comparable { final ffi.Pointer _ffi; // These are "used" in the sense that they keep dependencies alive @@ -226,19 +226,24 @@ final class Locale implements ffi.Finalizable { } /// See the [Rust documentation for `strict_cmp`](https://docs.rs/icu/latest/icu/locid/struct.Locale.html#method.strict_cmp) for more information. - Ordering strictCmp(String other) { + int compareToString(String other) { final temp = ffi2.Arena(); final otherView = other.utf8View; - final result = _ICU4XLocale_strict_cmp(_ffi, otherView.allocIn(temp), otherView.length); + final result = _ICU4XLocale_strict_cmp_(_ffi, otherView.allocIn(temp), otherView.length); temp.releaseAll(); - return Ordering.values.firstWhere((v) => v._ffi == result); + return result; } /// See the [Rust documentation for `total_cmp`](https://docs.rs/icu/latest/icu/locid/struct.Locale.html#method.total_cmp) for more information. - Ordering totalCmp(Locale other) { - final result = _ICU4XLocale_total_cmp(_ffi, other._ffi); - return Ordering.values.firstWhere((v) => v._ffi == result); + int compareTo(Locale other) { + final result = _ICU4XLocale_total_cmp_(_ffi, other._ffi); + return result; } + + @override + bool operator ==(Object other) => other is Locale && compareTo(other) == 0; + @override + int get hashCode => 42; // Cannot get hash from Rust, so a constant is the only correct impl } @meta.ResourceIdentifier('ICU4XLocale_destroy') @@ -316,12 +321,12 @@ external _ResultVoidInt32 _ICU4XLocale_to_string(ffi.Pointer self, f // ignore: non_constant_identifier_names external bool _ICU4XLocale_normalizing_eq(ffi.Pointer self, ffi.Pointer otherData, int otherLength); -@meta.ResourceIdentifier('ICU4XLocale_strict_cmp') -@ffi.Native, ffi.Pointer, ffi.Size)>(isLeaf: true, symbol: 'ICU4XLocale_strict_cmp') +@meta.ResourceIdentifier('ICU4XLocale_strict_cmp_') +@ffi.Native, ffi.Pointer, ffi.Size)>(isLeaf: true, symbol: 'ICU4XLocale_strict_cmp_') // ignore: non_constant_identifier_names -external int _ICU4XLocale_strict_cmp(ffi.Pointer self, ffi.Pointer otherData, int otherLength); +external int _ICU4XLocale_strict_cmp_(ffi.Pointer self, ffi.Pointer otherData, int otherLength); -@meta.ResourceIdentifier('ICU4XLocale_total_cmp') -@ffi.Native, ffi.Pointer)>(isLeaf: true, symbol: 'ICU4XLocale_total_cmp') +@meta.ResourceIdentifier('ICU4XLocale_total_cmp_') +@ffi.Native, ffi.Pointer)>(isLeaf: true, symbol: 'ICU4XLocale_total_cmp_') // ignore: non_constant_identifier_names -external int _ICU4XLocale_total_cmp(ffi.Pointer self, ffi.Pointer other); +external int _ICU4XLocale_total_cmp_(ffi.Pointer self, ffi.Pointer other); diff --git a/ffi/capi/bindings/dart/Ordering.g.dart b/ffi/capi/bindings/dart/Ordering.g.dart deleted file mode 100644 index 0b4ad1c6b51..00000000000 --- a/ffi/capi/bindings/dart/Ordering.g.dart +++ /dev/null @@ -1,23 +0,0 @@ -// generated by diplomat-tool - -part of 'lib.g.dart'; - -/// See the [Rust documentation for `Ordering`](https://docs.rs/core/latest/core/cmp/enum.Ordering.html) for more information. -enum Ordering { - less, - - equal, - - greater; - - int get _ffi { - switch (this) { - case less: - return -1; - case equal: - return 0; - case greater: - return 1; - } - } -} diff --git a/ffi/capi/bindings/dart/lib.g.dart b/ffi/capi/bindings/dart/lib.g.dart index 7e83b7ce4bc..897e5ff7e8c 100644 --- a/ffi/capi/bindings/dart/lib.g.dart +++ b/ffi/capi/bindings/dart/lib.g.dart @@ -79,7 +79,6 @@ part 'LineBreakOptions.g.dart'; part 'LineBreakStrictness.g.dart'; part 'LineBreakWordOption.g.dart'; part 'LineSegmenter.g.dart'; -part 'List.g.dart'; part 'ListFormatter.g.dart'; part 'ListLength.g.dart'; part 'Locale.g.dart'; @@ -96,7 +95,6 @@ part 'LocaleFallbacker.g.dart'; part 'LocaleFallbackerWithConfig.g.dart'; part 'Logger.g.dart'; part 'MetazoneCalculator.g.dart'; -part 'Ordering.g.dart'; part 'PluralCategories.g.dart'; part 'PluralCategory.g.dart'; part 'PluralOperands.g.dart'; @@ -214,6 +212,13 @@ extension on String { _Utf16View get utf16View => _Utf16View(this); } +extension on core.List { + // ignore: unused_element + _ListUtf8View get utf8View => _ListUtf8View(this); + // ignore: unused_element + _ListUtf16View get utf16View => _ListUtf16View(this); +} + extension on core.List { // ignore: unused_element _BoolListView get boolView => _BoolListView(this); @@ -279,7 +284,48 @@ class _Utf16View { } // ignore: unused_element -class _BoolListView{ +class _ListUtf8View { + final core.List _strings; + + // Copies + _ListUtf8View(this._strings); + + ffi.Pointer<_SliceUtf8> allocIn(ffi.Allocator alloc) { + final slice = alloc<_SliceUtf8>(length); + for (var i = 0; i < length; i++) { + final codeUnits = Utf8Encoder().convert(_strings[i]); + final str = alloc(codeUnits.length)..asTypedList(codeUnits.length).setRange(0, codeUnits.length, codeUnits); + slice[i]._data = str; + slice[i]._length = codeUnits.length; + } + return slice; + } + + int get length => _strings.length; +} + +// ignore: unused_element +class _ListUtf16View { + final core.List _strings; + + _ListUtf16View(this._strings); + + ffi.Pointer<_SliceUtf16> allocIn(ffi.Allocator alloc) { + final slice = alloc<_SliceUtf16>(length); + for (var i = 0; i < length; i++) { + final codeUnits = _strings[i].codeUnits; + final str = alloc(codeUnits.length)..asTypedList(codeUnits.length).setRange(0, codeUnits.length, codeUnits); + slice[i]._data = str; + slice[i]._length = codeUnits.length; + } + return slice; + } + + int get length => _strings.length; +} + +// ignore: unused_element +class _BoolListView { final core.List _values; _BoolListView(this._values); @@ -613,6 +659,40 @@ final class _SliceUsize extends ffi.Struct { } } +final class _SliceUtf16 extends ffi.Struct { + external ffi.Pointer _data; + + @ffi.Size() + external int _length; + + // This is expensive + @override + bool operator ==(Object other) { + if (other is! _SliceUtf16 || other._length != _length) { + return false; + } + + for (var i = 0; i < _length; i++) { + if (other._data[i] != _data[i]) { + return false; + } + } + return true; + } + + // This is cheap + @override + int get hashCode => _length.hashCode; + + String _toDart(core.List lifetimeEdges) { + final r = core.String.fromCharCodes(_data.asTypedList(_length)); + if (lifetimeEdges.isEmpty) { + _diplomat_free(_data.cast(), _length * 2, 2); + } + return r; + } +} + final class _SliceUtf8 extends ffi.Struct { external ffi.Pointer _data; diff --git a/ffi/capi/bindings/js/ICU4XLocaleFallbackIterator.d.ts b/ffi/capi/bindings/js/ICU4XLocaleFallbackIterator.d.ts index adfb952453f..348d2a7386c 100644 --- a/ffi/capi/bindings/js/ICU4XLocaleFallbackIterator.d.ts +++ b/ffi/capi/bindings/js/ICU4XLocaleFallbackIterator.d.ts @@ -23,10 +23,4 @@ export class ICU4XLocaleFallbackIterator { * See the {@link https://docs.rs/icu/latest/icu/locid_transform/fallback/struct.LocaleFallbackIterator.html#method.step Rust documentation for `step`} for more information. */ step(): void; - - /** - - * A combination of `get` and `step`. Returns the value that `get` would return and advances the iterator until hitting `und`. - */ - next(): ICU4XLocale | undefined; } diff --git a/ffi/capi/bindings/js/ICU4XLocaleFallbackIterator.mjs b/ffi/capi/bindings/js/ICU4XLocaleFallbackIterator.mjs index 7297bf7face..39f28a17b0a 100644 --- a/ffi/capi/bindings/js/ICU4XLocaleFallbackIterator.mjs +++ b/ffi/capi/bindings/js/ICU4XLocaleFallbackIterator.mjs @@ -23,11 +23,4 @@ export class ICU4XLocaleFallbackIterator { step() { wasm.ICU4XLocaleFallbackIterator_step(this.underlying); } - - next() { - return (() => { - const option_ptr = wasm.ICU4XLocaleFallbackIterator_next(this.underlying); - return (option_ptr == 0) ? undefined : new ICU4XLocale(option_ptr, true, []); - })(); - } } diff --git a/ffi/capi/src/collator.rs b/ffi/capi/src/collator.rs index 2e6080b5e34..c49341d0645 100644 --- a/ffi/capi/src/collator.rs +++ b/ffi/capi/src/collator.rs @@ -151,11 +151,26 @@ pub mod ffi { /// Ill-formed input is treated as if errors had been replaced with REPLACEMENT CHARACTERs according /// to the WHATWG Encoding Standard. #[diplomat::rust_link(icu::collator::Collator::compare_utf16, FnInStruct)] - #[diplomat::attr(dart, rename = "compare")] + #[diplomat::attr(dart, disable)] pub fn compare_utf16(&self, left: &DiplomatStr16, right: &DiplomatStr16) -> ICU4XOrdering { self.0.compare_utf16(left, right).into() } + /// Compare two strings. + /// + /// Ill-formed input is treated as if errors had been replaced with REPLACEMENT CHARACTERs according + /// to the WHATWG Encoding Standard. + #[diplomat::rust_link(icu::collator::Collator::compare_utf16, FnInStruct)] + #[diplomat::skip_if_ast] + #[diplomat::attr(dart, rename = "compare")] + pub fn compare_utf16_( + &self, + left: &DiplomatStr16, + right: &DiplomatStr16, + ) -> core::cmp::Ordering { + self.0.compare_utf16(left, right) + } + /// The resolved options showing how the default options, the requested options, /// and the options from locale data were combined. None of the struct fields /// will have `Auto` as the value. diff --git a/ffi/capi/src/common.rs b/ffi/capi/src/common.rs index d11ca260440..af493606300 100644 --- a/ffi/capi/src/common.rs +++ b/ffi/capi/src/common.rs @@ -8,6 +8,7 @@ pub mod ffi { #[diplomat::enum_convert(core::cmp::Ordering)] #[diplomat::rust_link(core::cmp::Ordering, Enum)] + #[diplomat::attr(*, disable)] pub enum ICU4XOrdering { Less = -1, Equal = 0, diff --git a/ffi/capi/src/decimal.rs b/ffi/capi/src/decimal.rs index f5450165791..c7b8d02fad7 100644 --- a/ffi/capi/src/decimal.rs +++ b/ffi/capi/src/decimal.rs @@ -67,7 +67,7 @@ pub mod ffi { /// /// The contents of the data struct will be consumed: if you wish to use the struct again it will have to be reconstructed. /// Passing a consumed struct to this method will return an error. - #[diplomat::attr(supports = constructors, disable)] + #[diplomat::attr(*, disable)] pub fn create_with_decimal_symbols_v1( data_struct: &ICU4XDataStruct, grouping_strategy: ICU4XFixedDecimalGroupingStrategy, diff --git a/ffi/capi/src/fallbacker.rs b/ffi/capi/src/fallbacker.rs index 061cb8ad882..731c0f93501 100644 --- a/ffi/capi/src/fallbacker.rs +++ b/ffi/capi/src/fallbacker.rs @@ -178,6 +178,7 @@ pub mod ffi { /// A combination of `get` and `step`. Returns the value that `get` would return /// and advances the iterator until hitting `und`. #[diplomat::attr(supports = iterators, iterator)] + #[diplomat::skip_if_ast] pub fn next(&mut self) -> Option> { let current = self.get(); if current.0 == icu_locid::Locale::UND { diff --git a/ffi/capi/src/lib.rs b/ffi/capi/src/lib.rs index 251b80fdd41..1ab59474437 100644 --- a/ffi/capi/src/lib.rs +++ b/ffi/capi/src/lib.rs @@ -66,6 +66,7 @@ pub mod locale; pub mod logging; #[macro_use] pub mod provider; +mod utf; // Components diff --git a/ffi/capi/src/list.rs b/ffi/capi/src/list.rs index a467b58a9c0..7f1bcc857bc 100644 --- a/ffi/capi/src/list.rs +++ b/ffi/capi/src/list.rs @@ -15,6 +15,7 @@ pub mod ffi { /// A list of strings #[diplomat::opaque] + #[diplomat::attr(*, disable)] pub struct ICU4XList(pub Vec); impl ICU4XList { @@ -122,6 +123,7 @@ pub mod ffi { #[diplomat::rust_link(icu::list::ListFormatter::format, FnInStruct)] #[diplomat::rust_link(icu::list::ListFormatter::format_to_string, FnInStruct, hidden)] #[diplomat::rust_link(icu::list::FormattedList, Struct, hidden)] + #[diplomat::attr(*, disable)] pub fn format( &self, list: &ICU4XList, @@ -130,5 +132,55 @@ pub mod ffi { self.0.format(list.0.iter()).write_to(write)?; Ok(()) } + + #[diplomat::rust_link(icu::list::ListFormatter::format, FnInStruct)] + #[diplomat::rust_link(icu::list::ListFormatter::format_to_string, FnInStruct, hidden)] + #[diplomat::rust_link(icu::list::FormattedList, Struct, hidden)] + #[diplomat::attr(dart, disable)] + #[diplomat::skip_if_ast] + pub fn format_valid_utf8( + &self, + list: &[&str], + write: &mut DiplomatWriteable, + ) -> Result<(), ICU4XError> { + self.0.format(list.iter()).write_to(write)?; + Ok(()) + } + + #[diplomat::rust_link(icu::list::ListFormatter::format, FnInStruct)] + #[diplomat::rust_link(icu::list::ListFormatter::format_to_string, FnInStruct, hidden)] + #[diplomat::rust_link(icu::list::FormattedList, Struct, hidden)] + #[diplomat::attr(dart, disable)] + #[diplomat::skip_if_ast] + pub fn format_utf8( + &self, + list: &[&DiplomatStr], + write: &mut DiplomatWriteable, + ) -> Result<(), ICU4XError> { + self.0 + .format(list.iter().copied().map(crate::utf::PotentiallyInvalidUtf8)) + .write_to(write)?; + Ok(()) + } + + #[diplomat::rust_link(icu::list::ListFormatter::format, FnInStruct)] + #[diplomat::rust_link(icu::list::ListFormatter::format_to_string, FnInStruct, hidden)] + #[diplomat::rust_link(icu::list::FormattedList, Struct, hidden)] + #[diplomat::attr(dart, rename = "format")] + #[diplomat::skip_if_ast] + pub fn format_utf16( + &self, + list: &[&DiplomatStr16], + write: &mut DiplomatWriteable, + ) -> Result<(), ICU4XError> { + self.0 + .format( + list.iter() + .copied() + .map(crate::utf::PotentiallyInvalidUtf16), + ) + .write_to(write)?; + Ok(()) + } } } diff --git a/ffi/capi/src/locale.rs b/ffi/capi/src/locale.rs index 55d6e3a5613..cb210b53c60 100644 --- a/ffi/capi/src/locale.rs +++ b/ffi/capi/src/locale.rs @@ -184,15 +184,31 @@ pub mod ffi { } #[diplomat::rust_link(icu::locid::Locale::strict_cmp, FnInStruct)] + #[diplomat::attr(*, disable)] pub fn strict_cmp(&self, other: &DiplomatStr) -> ICU4XOrdering { self.0.strict_cmp(other).into() } + #[diplomat::rust_link(icu::locid::Locale::strict_cmp, FnInStruct)] + #[diplomat::skip_if_ast] + #[diplomat::attr(dart, rename = "compareToString")] + pub fn strict_cmp_(&self, other: &DiplomatStr) -> core::cmp::Ordering { + self.0.strict_cmp(other) + } + #[diplomat::rust_link(icu::locid::Locale::total_cmp, FnInStruct)] + #[diplomat::attr(*, disable)] pub fn total_cmp(&self, other: &Self) -> ICU4XOrdering { self.0.total_cmp(&other.0).into() } + #[diplomat::rust_link(icu::locid::Locale::total_cmp, FnInStruct)] + #[diplomat::skip_if_ast] + #[diplomat::attr(supports = comparators, comparison)] + pub fn total_cmp_(&self, other: &Self) -> core::cmp::Ordering { + self.0.total_cmp(&other.0) + } + /// Deprecated /// /// Use `create_from_string("en"). diff --git a/ffi/capi/src/utf.rs b/ffi/capi/src/utf.rs new file mode 100644 index 00000000000..b9ed64d5830 --- /dev/null +++ b/ffi/capi/src/utf.rs @@ -0,0 +1,122 @@ +// This file is part of ICU4X. For terms of use, please see the file +// called LICENSE at the top level of the ICU4X source tree +// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). + +use alloc::borrow::Cow; + +use writeable::{impl_display_with_writeable, LengthHint, Writeable}; + +use core::fmt; + +/// Implements [`Writeable`] for [`&[u8]`] according to the [WHATWG Encoding Standard]( +/// https://encoding.spec.whatwg.org/#utf-8-decoder). +#[derive(Debug)] +#[allow(clippy::exhaustive_structs)] // newtype +pub struct PotentiallyInvalidUtf8<'a>(pub &'a [u8]); + +impl Writeable for PotentiallyInvalidUtf8<'_> { + fn write_to(&self, sink: &mut W) -> fmt::Result { + let mut remaining = self.0; + loop { + match core::str::from_utf8(remaining) { + Ok(valid) => { + return sink.write_str(valid); + } + Err(e) => { + // SAFETY: By Utf8Error invariants + let valid = unsafe { + core::str::from_utf8_unchecked(remaining.get_unchecked(..e.valid_up_to())) + }; + sink.write_str(valid)?; + sink.write_char(char::REPLACEMENT_CHARACTER)?; + let Some(error_len) = e.error_len() else { + return Ok(()); // end of string + }; + // SAFETY: By Utf8Error invariants + remaining = unsafe { remaining.get_unchecked(e.valid_up_to() + error_len..) } + } + } + } + } + + fn writeable_length_hint(&self) -> writeable::LengthHint { + // Lower bound is all valid UTF-8, upper bound is all bytes with the high bit, which become replacement characters. + LengthHint::between(self.0.len(), self.0.len() * 3) + } + + fn write_to_string(&self) -> Cow { + match core::str::from_utf8(self.0) { + Ok(valid) => Cow::Borrowed(valid), + Err(e) => { + // SAFETY: By Utf8Error invariants + let valid = unsafe { + core::str::from_utf8_unchecked(self.0.get_unchecked(..e.valid_up_to())) + }; + + // Let's assume this is the only error + let mut out = alloc::string::String::with_capacity( + self.0.len() + char::REPLACEMENT_CHARACTER.len_utf8() + - e.error_len().unwrap_or(0), + ); + + out.push_str(valid); + out.push('�'); + + // If there's more, we can use `write_to` + if let Some(error_len) = e.error_len() { + // SAFETY: By Utf8Error invariants + let remaining = unsafe { self.0.get_unchecked(e.valid_up_to() + error_len..) }; + let _infallible = Self(remaining).write_to(&mut out); + } + out.into() + } + } + } +} + +impl_display_with_writeable!(PotentiallyInvalidUtf8<'_>); + +/// Implements [`Writeable`] for [`&[u16]`] according to the [WHATWG Encoding Standard]( +/// https://encoding.spec.whatwg.org/#shared-utf-16-decoder). +#[derive(Debug)] +#[allow(clippy::exhaustive_structs)] // newtype +pub struct PotentiallyInvalidUtf16<'a>(pub &'a [u16]); + +impl Writeable for PotentiallyInvalidUtf16<'_> { + fn write_to(&self, sink: &mut W) -> fmt::Result { + for c in core::char::decode_utf16(self.0.iter().copied()) { + if let Ok(c) = c { + sink.write_char(c)?; + } else { + sink.write_char(char::REPLACEMENT_CHARACTER)?; + } + } + Ok(()) + } + + fn writeable_length_hint(&self) -> LengthHint { + // Lower bound is all ASCII, upper bound is all 3-byte code points (including replacement character) + LengthHint::between(self.0.len(), self.0.len() * 3) + } +} + +impl_display_with_writeable!(PotentiallyInvalidUtf16<'_>); + +#[cfg(test)] +mod test { + use super::*; + use writeable::assert_writeable_eq; + + #[test] + fn test_utf8() { + assert_writeable_eq!(PotentiallyInvalidUtf8(b"Foo Bar"), "Foo Bar"); + assert_writeable_eq!(PotentiallyInvalidUtf8(b"Foo\xFDBar"), "Foo�Bar"); + assert_writeable_eq!(PotentiallyInvalidUtf8(b"Foo\xFDBar\xff"), "Foo�Bar�"); + } + + #[test] + fn test_utf16() { + assert_writeable_eq!(PotentiallyInvalidUtf16(&[0xD83E, 0xDD73]), "🥳"); + assert_writeable_eq!(PotentiallyInvalidUtf16(&[0xD83E, 0x20, 0xDD73]), "� �"); + } +} diff --git a/ffi/dart/test/icu_test.dart b/ffi/dart/test/icu_test.dart index 7c098b5edfe..ed6bd63c150 100644 --- a/ffi/dart/test/icu_test.dart +++ b/ffi/dart/test/icu_test.dart @@ -17,13 +17,13 @@ void main() { fallbackSupplement: LocaleFallbackSupplement.none)) .fallbackForLocale(Locale.fromString('de-CH-u-ca-japanese')); expect(iterator.moveNext(), true); - expect(iterator.current.toString(), 'de-CH-u-ca-japanese'); + expect(iterator.current, Locale.fromString('de-CH-u-ca-japanese')); expect(iterator.moveNext(), true); - expect(iterator.current.toString(), 'de-CH'); + expect(iterator.current, Locale.fromString('de-CH')); expect(iterator.moveNext(), true); - expect(iterator.current.toString(), 'und-CH-u-ca-japanese'); + expect(iterator.current, Locale.fromString('und-CH-u-ca-japanese')); expect(iterator.moveNext(), true); - expect(iterator.current.toString(), 'und-CH'); + expect(iterator.current, Locale.fromString('und-CH')); expect(iterator.moveNext(), false); }); @@ -42,12 +42,22 @@ void main() { test('ListFormatter', () { final formatter = ListFormatter.andWithLength( DataProvider.compiled(), Locale.fromString('es'), ListLength.wide); - final list = List() - ..push('España') - ..push('Francia') - ..push('Suiza') - ..push('Italia'); + final list = ['España', 'Francia', 'Suiza', 'Italia']; expect(formatter.format(list), 'España, Francia, Suiza e Italia'); }); + + test('Locale ordering', () { + expect( + [ + Locale.fromString('en-GB'), + Locale.fromString('de'), + Locale.fromString('az'), + ]..sort(), + [ + Locale.fromString('az'), + Locale.fromString('de'), + Locale.fromString('en-GB'), + ]); + }); } diff --git a/utils/writeable/src/lib.rs b/utils/writeable/src/lib.rs index 7d52288945a..bdf7b738ed4 100644 --- a/utils/writeable/src/lib.rs +++ b/utils/writeable/src/lib.rs @@ -397,19 +397,21 @@ macro_rules! assert_writeable_eq { ($actual_writeable:expr, $expected_str:expr, $($arg:tt)+) => {{ let actual_writeable = &$actual_writeable; let (actual_str, _) = $crate::writeable_to_parts_for_test(actual_writeable).unwrap(); + let actual_len = actual_str.len(); assert_eq!(actual_str, $expected_str, $($arg)*); assert_eq!(actual_str, $crate::Writeable::write_to_string(actual_writeable), $($arg)+); let length_hint = $crate::Writeable::writeable_length_hint(actual_writeable); + let lower = length_hint.0; assert!( - length_hint.0 <= actual_str.len(), - "hint lower bound {} larger than actual length {}: {}", - length_hint.0, actual_str.len(), format!($($arg)*), + lower <= actual_len, + "hint lower bound {lower} larger than actual length {actual_len}: {}", + format!($($arg)*), ); if let Some(upper) = length_hint.1 { assert!( - actual_str.len() <= upper, - "hint upper bound {} smaller than actual length {}: {}", - length_hint.0, actual_str.len(), format!($($arg)*), + actual_len <= upper, + "hint upper bound {upper} smaller than actual length {actual_len}: {}", + format!($($arg)*), ); } assert_eq!(actual_writeable.to_string(), $expected_str); @@ -425,13 +427,23 @@ macro_rules! assert_writeable_parts_eq { ($actual_writeable:expr, $expected_str:expr, $expected_parts:expr, $($arg:tt)+) => {{ let actual_writeable = &$actual_writeable; let (actual_str, actual_parts) = $crate::writeable_to_parts_for_test(actual_writeable).unwrap(); + let actual_len = actual_str.len(); assert_eq!(actual_str, $expected_str, $($arg)+); assert_eq!(actual_str, $crate::Writeable::write_to_string(actual_writeable), $($arg)+); assert_eq!(actual_parts, $expected_parts, $($arg)+); let length_hint = $crate::Writeable::writeable_length_hint(actual_writeable); - assert!(length_hint.0 <= actual_str.len(), $($arg)+); + let lower = length_hint.0; + assert!( + lower <= actual_len, + "hint lower bound {lower} larger than actual length {actual_len}: {}", + format!($($arg)*), + ); if let Some(upper) = length_hint.1 { - assert!(actual_str.len() <= upper, $($arg)+); + assert!( + actual_len <= upper, + "hint upper bound {upper} smaller than actual length {actual_len}: {}", + format!($($arg)*), + ); } assert_eq!(actual_writeable.to_string(), $expected_str); }};