Add size-optimized FFI functions for Locale and DataProvider

.github/workflows/build-test.yml

@@ -112,10 +112,11 @@ jobs:
       run: rustup show
     - name: Load cortex target for no_std build.
       run: |
-        rustup install nightly
-        rustup component add --toolchain nightly rust-src
-        rustup target add thumbv7m-none-eabi --toolchain nightly
-        rustup target add thumbv8m.main-none-eabihf --toolchain nightly
+        rustup install nightly-2021-02-28
+        rustup component add --toolchain nightly-2021-02-28 rust-src
+        rustup target add thumbv7m-none-eabi --toolchain nightly-2021-02-28
+        rustup target add thumbv8m.main-none-eabihf --toolchain nightly-2021-02-28
+        rustup target add x86_64-unknown-linux-gnu --toolchain nightly-2021-02-28
     - name: Get cargo-make version
       id: cargo-make-version
       run: |

ffi/capi/Cargo.toml

@@ -61,8 +61,10 @@ fixed_decimal = { path = "../../utils/fixed_decimal" }
 icu_decimal = { path = "../../components/decimal/" }
 icu_locale_canonicalizer = { path = "../../components/locale_canonicalizer" }
 icu_locid = { path = "../../components/locid" }
+icu_locid_macros = { path = "../../components/locid/macros" }
 icu_plurals = { path = "../../components/plurals/" }
 icu_provider = { path = "../../provider/core", features = ["provider_serde"] }
+icu_provider_blob = { path = "../../provider/blob" }
 tinystr = { version = "0.4.10", features = ["alloc"], default-features = false }
 writeable = { path = "../../utils/writeable/" }
 

ffi/capi/examples/fixeddecimal/.gitignore

@@ -1,6 +1 @@
 a.out
-optim*
-*.i64
-*.dot
-*.elf
-*.o

ffi/capi/examples/fixeddecimal/Makefile

@@ -16,54 +16,10 @@ $(ALL_HEADERS):
 ../../../../target/debug/libicu_capi.a: $(ALL_RUST)
 	cargo build
 
-../../../../target/x86_64-unknown-linux-gnu/debug/libicu_capi.a: $(ALL_RUST)
-	RUSTFLAGS="-Clinker-plugin-lto -Clinker=clang -Clink-arg=-flto -Cpanic=abort" cargo +nightly-2021-02-28 panic-abort-build --target x86_64-unknown-linux-gnu --no-default-features --features x86tiny
-
-../../../../target/x86_64-unknown-linux-gnu/release/libicu_capi.a: $(ALL_RUST)
-	RUSTFLAGS="-Clinker-plugin-lto -Clinker=clang -Clink-arg=-flto -Cpanic=abort" cargo +nightly-2021-02-28 panic-abort-build --target x86_64-unknown-linux-gnu --no-default-features --features x86tiny --features smaller_static --release
-
-# Naive target: no optimizations, full std
 a.out: ../../../../target/debug/libicu_capi.a $(ALL_HEADERS) test.c
 	gcc test.c ../../../../target/debug/libicu_capi.a -ldl -lpthread -lm -g
 
-# optim.elf: gcc with maximum link-time code stripping (gc-sections and strip-all)
-optim.elf: ../../../../target/debug/libicu_capi.a $(ALL_HEADERS) test.c
-	gcc -fdata-sections -ffunction-sections test.c ../../../../target/debug/libicu_capi.a -ldl -lpthread -lm -g -o optim.elf -Wl,--gc-sections -Wl,--strip-all
-
-# optim2.elf: clang single-step with gc-sections
-optim2.elf: ../../../../target/x86_64-unknown-linux-gnu/debug/libicu_capi.a $(ALL_HEADERS) test.c
-	clang -flto -fdata-sections -ffunction-sections test.c ../../../../target/x86_64-unknown-linux-gnu/debug/libicu_capi.a -g -o optim2.elf -Wl,--gc-sections
-
-optim3.o: $(ALL_HEADERS)
-	clang -c -flto=thin -fdata-sections -ffunction-sections --target=x86_64-unknown-linux-gnu test.c -g -o optim3.o
-
-# optim3.elf: clang two-step with lld, debug mode
-optim3.elf: optim3.o ../../../../target/x86_64-unknown-linux-gnu/debug/libicu_capi.a
-	clang -flto=thin -fuse-ld=lld -L . -o optim3.elf optim3.o ../../../../target/x86_64-unknown-linux-gnu/debug/libicu_capi.a -Wl,--gc-sections
-
-optim4.o: $(ALL_HEADERS)
-	clang -c -flto=thin -fdata-sections -ffunction-sections --target=x86_64-unknown-linux-gnu test.c -g -o optim4.o
-
-# optim4.elf: clang two-step with lld, release mode with debug symbols
-optim4.elf: optim4.o ../../../../target/x86_64-unknown-linux-gnu/release/libicu_capi.a
-	clang -flto=thin -fuse-ld=lld -L . -o optim4.elf optim4.o ../../../../target/x86_64-unknown-linux-gnu/release/libicu_capi.a -Wl,--gc-sections
-
-optim5.o: $(ALL_HEADERS)
-	clang -c -flto=thin -fdata-sections -ffunction-sections --target=x86_64-unknown-linux-gnu test.c -o optim5.o
-
-# optim5.elf: clang two-step with lld, release mode stripped of debug symbols
-optim5.elf: optim5.o ../../../../target/x86_64-unknown-linux-gnu/release/libicu_capi.a
-	clang -flto=thin -fuse-ld=lld -L . -o optim5.elf optim5.o ../../../../target/x86_64-unknown-linux-gnu/release/libicu_capi.a -Wl,--gc-sections -Wl,--strip-all
-
 build: a.out
 
 test: build
 	./a.out
-
-build-optim: optim.elf optim2.elf optim3.elf optim4.elf optim5.elf
-
-# note: optim2.elf and optim3.elf crash when run with error "Illegal instruction" (investigate?)
-test-optim: build-optim
-	./optim.elf
-	./optim4.elf
-	./optim5.elf

ffi/capi/examples/fixeddecimal_tiny/.gitignore

@@ -0,0 +1,6 @@
+a.out
+optim*
+*.i64
+*.dot
+*.elf
+*.o

ffi/capi/examples/fixeddecimal_tiny/Makefile

@@ -0,0 +1,65 @@
+# This file is part of ICU4X. For terms of use, please see the file
+# called LICENSE at the top level of the ICU4X source tree
+# (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+.DEFAULT_GOAL := test
+.PHONY: build test
+
+ALL_HEADERS := $(wildcard ../../include/*.h)
+ALL_RUST := $(wildcard ../../src/*.rs)
+
+$(ALL_RUST):
+
+$(ALL_HEADERS):
+
+
+../../../../target/debug/libicu_capi.a: $(ALL_RUST)
+	cargo build
+
+../../../../target/x86_64-unknown-linux-gnu/debug/libicu_capi.a: $(ALL_RUST)
+	RUSTFLAGS="-Clinker-plugin-lto -Clinker=clang -Clink-arg=-flto -Cpanic=abort" cargo +nightly-2021-02-28 panic-abort-build --target x86_64-unknown-linux-gnu --no-default-features --features x86tiny
+
+../../../../target/x86_64-unknown-linux-gnu/release/libicu_capi.a: $(ALL_RUST)
+	RUSTFLAGS="-Clinker-plugin-lto -Clinker=clang -Clink-arg=-flto -Cpanic=abort" cargo +nightly-2021-02-28 panic-abort-build --target x86_64-unknown-linux-gnu --no-default-features --features x86tiny --features smaller_static --release
+
+# Naive target: no optimizations, full std
+optim0.elf: ../../../../target/debug/libicu_capi.a $(ALL_HEADERS) test.c
+	gcc test.c ../../../../target/debug/libicu_capi.a -ldl -lpthread -lm -g -o optim0.elf
+
+# optim.elf: gcc with maximum link-time code stripping (gc-sections and strip-all)
+optim1.elf: ../../../../target/debug/libicu_capi.a $(ALL_HEADERS) test.c
+	gcc -fdata-sections -ffunction-sections test.c ../../../../target/debug/libicu_capi.a -ldl -lpthread -lm -g -o optim1.elf -Wl,--gc-sections -Wl,--strip-all
+
+# optim2.elf: clang single-step with gc-sections
+optim2.elf: ../../../../target/x86_64-unknown-linux-gnu/debug/libicu_capi.a $(ALL_HEADERS) test.c
+	clang -flto -fdata-sections -ffunction-sections test.c ../../../../target/x86_64-unknown-linux-gnu/debug/libicu_capi.a -g -o optim2.elf -Wl,--gc-sections
+
+optim3.o: $(ALL_HEADERS)
+	clang -c -flto=thin -fdata-sections -ffunction-sections --target=x86_64-unknown-linux-gnu test.c -g -o optim3.o
+
+# optim3.elf: clang two-step with lld, debug mode
+optim3.elf: optim3.o ../../../../target/x86_64-unknown-linux-gnu/debug/libicu_capi.a
+	clang -flto=thin -fuse-ld=lld -L . -o optim3.elf optim3.o ../../../../target/x86_64-unknown-linux-gnu/debug/libicu_capi.a -Wl,--gc-sections
+
+optim4.o: $(ALL_HEADERS)
+	clang -c -flto=thin -fdata-sections -ffunction-sections --target=x86_64-unknown-linux-gnu test.c -g -o optim4.o
+
+# optim4.elf: clang two-step with lld, release mode with debug symbols
+optim4.elf: optim4.o ../../../../target/x86_64-unknown-linux-gnu/release/libicu_capi.a
+	clang -flto=thin -fuse-ld=lld -L . -o optim4.elf optim4.o ../../../../target/x86_64-unknown-linux-gnu/release/libicu_capi.a -Wl,--gc-sections
+
+optim5.o: $(ALL_HEADERS)
+	clang -c -flto=thin -fdata-sections -ffunction-sections --target=x86_64-unknown-linux-gnu test.c -o optim5.o
+
+# optim5.elf: clang two-step with lld, release mode stripped of debug symbols
+optim5.elf: optim5.o ../../../../target/x86_64-unknown-linux-gnu/release/libicu_capi.a
+	clang -flto=thin -fuse-ld=lld -L . -o optim5.elf optim5.o ../../../../target/x86_64-unknown-linux-gnu/release/libicu_capi.a -Wl,--gc-sections -Wl,--strip-all
+
+build: optim0.elf optim1.elf optim2.elf optim3.elf optim4.elf optim5.elf
+
+# note: optim2.elf and optim3.elf crash when run with error "Illegal instruction" (investigate?)
+test: build
+	./optim0.elf
+	./optim1.elf
+	./optim4.elf
+	./optim5.elf

ffi/capi/examples/fixeddecimal_tiny/README.md

@@ -0,0 +1,15 @@
+# Tiny FixedDecimal FFI Demo
+
+This example contains tooling to build a size-optimized binary using FixedDecimal and FixedDecimalFormat in C over FFI.
+
+Prerequisites: `clang` and `lld`, which must be compatible with the Rust toolchain. `apt-get install clang lld` *might* work, but if you run into errors, refer to the following thread for tips:
+
+https://github.com/rust-lang/rust/issues/60059
+
+You also need to install the correct toolchains:
+
+```bash
+$ rustup install nightly-2021-02-28
+$ rustup component add --toolchain nightly-2021-02-28 rust-src
+$ rustup target add x86_64-unknown-linux-gnu --toolchain nightly-2021-02-28
+```

ffi/capi/examples/fixeddecimal_tiny/test.c

@@ -0,0 +1,50 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+#include "../../include/ICU4XFixedDecimalFormat.h"
+#include <string.h>
+#include <stdio.h>
+
+int main() {
+    ICU4XLocale* locale = ICU4XLocale_create_bn();
+    ICU4XCreateStaticDataProviderResult result = ICU4XStaticDataProvider_create();
+    if (!result.success) {
+        printf("Failed to create FsDataProvider\n");
+        return 1;
+    }
+    ICU4XStaticDataProvider* provider = result.provider;
+    ICU4XFixedDecimal* decimal = ICU4XFixedDecimal_create(1000007);
+
+    ICU4XFixedDecimalFormatOptions opts = {ICU4XFixedDecimalGroupingStrategy_Auto, ICU4XFixedDecimalSignDisplay_Auto};
+
+    ICU4XFixedDecimalFormatResult fdf_result = ICU4XFixedDecimalFormat_try_new_from_static(locale, provider, opts);
+    if (!fdf_result.success)  {
+        printf("Failed to create FixedDecimalFormat\n");
+        return 1;
+    }
+    ICU4XFixedDecimalFormat* fdf = fdf_result.fdf;
+    char output[40];
+
+    DiplomatWriteable write = diplomat_simple_writeable(output, 40);
+
+    bool success = ICU4XFixedDecimalFormat_format(fdf, decimal, &write).is_ok;
+    if (!success) {
+        printf("Failed to write result of FixedDecimalFormat::format to string.\n");
+        return 1;
+    }
+    printf("Output is %s\n", output);
+
+    const char* expected = u8"১০,০০,০০৭";
+    if (strcmp(output, expected) != 0) {
+        printf("Output does not match expected output!\n");
+        return 1;
+    }
+
+    ICU4XFixedDecimal_destroy(decimal);
+    ICU4XFixedDecimalFormat_destroy(fdf);
+    ICU4XLocale_destroy(locale);
+    ICU4XStaticDataProvider_destroy(provider);
+
+    return 0;
+}

ffi/capi/include/ICU4XCreateStaticDataProviderResult.h

@@ -0,0 +1,24 @@
+#ifndef ICU4XCreateStaticDataProviderResult_H
+#define ICU4XCreateStaticDataProviderResult_H
+#include <stdio.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+#include "diplomat_runtime.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+typedef struct ICU4XStaticDataProvider ICU4XStaticDataProvider;
+
+typedef struct ICU4XCreateStaticDataProviderResult {
+    ICU4XStaticDataProvider* provider;
+    bool success;
+} ICU4XCreateStaticDataProviderResult;
+
+void ICU4XCreateStaticDataProviderResult_destroy(ICU4XCreateStaticDataProviderResult* self);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

ffi/capi/include/ICU4XFixedDecimalFormat.h

@@ -15,11 +15,14 @@ typedef struct ICU4XFixedDecimalFormat ICU4XFixedDecimalFormat;
 #include "ICU4XDataProvider.h"
 #include "ICU4XFixedDecimalFormatOptions.h"
 #include "ICU4XFixedDecimalFormatResult.h"
+#include "ICU4XStaticDataProvider.h"
 #include "ICU4XFixedDecimal.h"
 #include "result_void_void.h"
 
 ICU4XFixedDecimalFormatResult ICU4XFixedDecimalFormat_try_new(const ICU4XLocale* locale, const ICU4XDataProvider* provider, ICU4XFixedDecimalFormatOptions options);
 
+ICU4XFixedDecimalFormatResult ICU4XFixedDecimalFormat_try_new_from_static(const ICU4XLocale* locale, const ICU4XStaticDataProvider* provider, ICU4XFixedDecimalFormatOptions options);
+
 decimal_ffi_result_void_void ICU4XFixedDecimalFormat_format(const ICU4XFixedDecimalFormat* self, const ICU4XFixedDecimal* value, DiplomatWriteable* write);
 void ICU4XFixedDecimalFormat_destroy(ICU4XFixedDecimalFormat* self);
 

ffi/capi/include/ICU4XLocale.h

@@ -15,6 +15,10 @@ typedef struct ICU4XLocale ICU4XLocale;
 
 ICU4XLocale* ICU4XLocale_create(const char* name_data, size_t name_len);
 
+ICU4XLocale* ICU4XLocale_create_en();
+
+ICU4XLocale* ICU4XLocale_create_bn();
+
 ICU4XLocale* ICU4XLocale_clone(const ICU4XLocale* self);
 
 locale_ffi_result_void_ICU4XLocaleError ICU4XLocale_basename(const ICU4XLocale* self, DiplomatWriteable* write);

ffi/capi/include/ICU4XStaticDataProvider.h

@@ -0,0 +1,22 @@
+#ifndef ICU4XStaticDataProvider_H
+#define ICU4XStaticDataProvider_H
+#include <stdio.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+#include "diplomat_runtime.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct ICU4XStaticDataProvider ICU4XStaticDataProvider;
+#include "ICU4XCreateStaticDataProviderResult.h"
+
+ICU4XCreateStaticDataProviderResult ICU4XStaticDataProvider_create();
+void ICU4XStaticDataProvider_destroy(ICU4XStaticDataProvider* self);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

ffi/capi/src/decimal.rs

@@ -8,13 +8,15 @@ pub mod ffi {
     use diplomat_runtime::DiplomatResult;
     use icu_decimal::{
         options::{FixedDecimalFormatOptions, GroupingStrategy, SignDisplay},
+        provider::DecimalSymbolsV1Marker,
         FixedDecimalFormat,
     };
+    use icu_provider::prelude::DataProvider;
     use writeable::Writeable;
 
     use crate::{
         fixed_decimal::ffi::ICU4XFixedDecimal, locale::ffi::ICU4XLocale,
-        provider::ffi::ICU4XDataProvider,
+        provider::ffi::ICU4XDataProvider, provider::ffi::ICU4XStaticDataProvider,
     };
 
     #[diplomat::opaque]
@@ -65,8 +67,30 @@ pub mod ffi {
             provider: &ICU4XDataProvider,
             options: ICU4XFixedDecimalFormatOptions,
         ) -> ICU4XFixedDecimalFormatResult {
-            let langid = locale.0.as_ref().clone();
             let provider = provider.0.as_ref();
+            Self::try_new_impl(locale, provider, options)
+        }
+
+        /// Creates a new [`ICU4XFixedDecimalFormat`] from a [`ICU4XStaticDataProvider`].
+        /// See [the Rust docs](https://unicode-org.github.io/icu4x-docs/doc/icu/decimal/struct.FixedDecimalFormat.html#method.try_new) for more information.
+        pub fn try_new_from_static(
+            locale: &ICU4XLocale,
+            provider: &ICU4XStaticDataProvider,
+            options: ICU4XFixedDecimalFormatOptions,
+        ) -> ICU4XFixedDecimalFormatResult {
+            let provider = provider.0.as_ref();
+            Self::try_new_impl(locale, provider, options)
+        }
+
+        fn try_new_impl<D>(
+            locale: &ICU4XLocale,
+            provider: &D,
+            options: ICU4XFixedDecimalFormatOptions,
+        ) -> ICU4XFixedDecimalFormatResult
+        where
+            D: DataProvider<'static, DecimalSymbolsV1Marker> + ?Sized,
+        {
+            let langid = locale.0.as_ref().clone();
 
             if let Result::Ok(fdf) = FixedDecimalFormat::try_new(
                 langid,

ffi/capi/src/locale.rs

@@ -31,6 +31,16 @@ pub mod ffi {
                 .map(|l| Box::new(ICU4XLocale(l)))
         }
 
+        /// Construct an [`ICU4XLocale`] for the English language.
+        pub fn create_en() -> Box<ICU4XLocale> {
+            Box::new(ICU4XLocale(icu_locid_macros::langid!("en").into()))
+        }
+
+        /// Construct an [`ICU4XLocale`] for the Bangla language.
+        pub fn create_bn() -> Box<ICU4XLocale> {
+            Box::new(ICU4XLocale(icu_locid_macros::langid!("bn").into()))
+        }
+
         /// Clones the [`ICU4XLocale`].
         /// See [the Rust docs](https://unicode-org.github.io/icu4x-docs/doc/icu/locid/struct.Locale.html) for more information.
         #[allow(clippy::should_implement_trait)]