Skip to content

Commit

Permalink
squash
Browse files Browse the repository at this point in the history
  • Loading branch information
@robertbastian
robertbastian committed Apr 26, 2023
1 parent a3f79c5 commit 115554b
Show file tree
Hide file tree
Showing 41 changed files with 1,436 additions and 972 deletions.
2 changes: 0 additions & 2 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion provider/blob/src/export/blob_exporter.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ impl DataExporter for BlobExporter<'_> {
locale: &DataLocale,
payload: &DataPayload<ExportMarker>,
) -> Result<(), DataError> {
log::trace!("Adding: {}/{}", key, locale);
let mut serializer = postcard::Serializer {
output: AllocVec::new(),
};
Expand Down
57 changes: 19 additions & 38 deletions provider/blob/src/export/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,60 +2,41 @@
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

//! Data generation for [`BlobDataProvider`](crate::BlobDataProvider) data. See the `icu_datagen` crate.
//! Data exporter for [`BlobDataProvider`](crate::BlobDataProvider).
//!
//! # Examples
//! This module can be used as a target for the `icu_datagen` crate.
//!
//! ```
//! use icu_provider::datagen::DataExporter;
//! use icu_provider::dynutil::*;
//! # Examples
//!
//! use icu_datagen::prelude::*;
//! use icu_provider::hello_world::*;
//! use icu_provider::prelude::*;
//! use icu_provider_blob::export::BlobExporter;
//! use icu_provider_blob::export::*;
//! use icu_provider_blob::BlobDataProvider;
//! use std::borrow::Cow;
//! use std::io::Read;
//! use std::rc::Rc;
//!
//! let mut buffer: Vec<u8> = Vec::new();
//!
//! let payload = DataPayload::<HelloWorldV1Marker>::from_owned(HelloWorldV1 {
//! message: Cow::Borrowed("Hi"),
//! });
//!
//! // Set up the exporter
//! let mut exporter = BlobExporter::new_with_sink(Box::new(&mut buffer));
//!
//! // Export something
//! {
//! let mut exporter = BlobExporter::new_with_sink(Box::new(&mut buffer));
//! exporter
//! .put_payload(
//! HelloWorldV1Marker::KEY,
//! &Default::default(),
//! &UpcastDataPayload::upcast(payload.clone()),
//! )
//! .expect("Should successfully export");
//! exporter
//! .close()
//! .expect("Should successfully dump to buffer");
//! }
//! DatagenProvider::default()
//! .export(
//! [HelloWorldV1Marker::KEY].into_iter().collect(),
//! exporter
//! ).unwrap();
//!
//! // Create a blob provider reading from the buffer
//! // Create a filesystem provider reading from the demo directory
//! let provider =
//! BlobDataProvider::try_new_from_blob(buffer.into_boxed_slice())
//! .expect("Should successfully read from buffer");
//! .expect("Should successfully read from buffer")
//! .as_deserializing();
//!
//! // Read the key from the filesystem and ensure it is as expected
//! let req = DataRequest {
//! locale: Default::default(),
//! metadata: Default::default(),
//! };
//! // Read the key from the blob
//! let response: DataPayload<HelloWorldV1Marker> = provider
//! .as_deserializing()
//! .load(req)
//! .load(Default::default())
//! .unwrap()
//! .take_payload()
//! .unwrap();
//!
//! assert_eq!(response.get(), payload.get(),);
//! ```

mod blob_exporter;
Expand Down
26 changes: 15 additions & 11 deletions provider/datagen/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,18 @@ icu_segmenter = { version = "1.2.0", path = "../../components/segmenter", featur
icu_timezone = { version = "1.2.0", path = "../../components/timezone", features = ["datagen"] }

# ICU provider infrastructure
icu_provider = { version = "1.2.0", path = "../core", features = ["std", "log_error_context", "datagen"]}
icu_provider = { version = "1.2.0", path = "../core", features = ["std", "log_error_context", "datagen", "deserialize_json"]}
icu_provider_adapters = { version = "1.2.0", path = "../adapters", features = ["datagen"] }
icu_provider_blob = { version = "1.2.0", path = "../blob", features = ["export"] }
icu_provider_fs = { version = "1.2.0", path = "../fs", features = ["export"] }

# Exporters
icu_provider_blob = { version = "1.2.0", path = "../blob", features = ["export"], optional = true }
icu_provider_fs = { version = "1.2.0", path = "../fs", features = ["export"], optional = true }
crlify = { version = "1.0.1", path = "../../utils/crlify", optional = true }
databake = { version = "0.1.3", path = "../../utils/databake", optional = true}
syn = {version = "1.0", features = ["parsing"], optional = true }

# Other
cached-path = { version = ">=0.5, <0.7", optional = true }
crlify = { version = "1.0.1", path = "../../utils/crlify"}
databake = { version = "0.1.3", path = "../../utils/databake"}
displaydoc = { version = "0.2.3", default-features = false }
elsa = "1.7"
icu_codepointtrie_builder = { version = "0.3.4", path = "../../components/collections/codepointtrie_builder", default-features = false }
Expand All @@ -66,13 +69,10 @@ itertools = "0.10"
lazy_static = "1"
log = "0.4"
ndarray = { version = "0.15.5", default-features = false }
proc-macro2 = "1.0"
quote = "1.0.9"
rayon = "1.5"
serde = { version = "1.0", default-features = false, features = ["derive", "alloc"] }
serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
serde-aux = { version = "4.1.2", default-features = false }
syn = {version = "1.0", features = ["parsing"] }
tinystr = { version = "0.7.1", path = "../../utils/tinystr", features = ["alloc", "serde", "zerovec"], default-features = false }
toml = "0.5"
writeable = { version = "0.5.1", path = "../../utils/writeable" }
Expand All @@ -95,7 +95,11 @@ repodata = { path = "../../provider/repodata" }
dhat = "0.3.0"

[features]
default = ["bin", "use_wasm", "networking"]
default = ["bin", "use_wasm", "networking", "legacy_api"]
provider_baked = ["dep:crlify", "dep:databake", "dep:syn"]
provider_blob = ["dep:icu_provider_blob"]
provider_fs = ["dep:icu_provider_fs"]
legacy_api = ["provider_fs", "provider_blob", "provider_baked"]
bin = ["dep:clap", "dep:eyre", "dep:simple_logger"]
# Use wasm for building codepointtries
use_wasm = ["icu_codepointtrie_builder/wasm"]
Expand All @@ -108,7 +112,7 @@ networking = ["dep:cached-path"]

[[bin]]
name = "icu4x-datagen"
path = "src/bin/datagen.rs"
path = "src/bin/datagen/mod.rs"
required-features = ["bin"]

[[test]]
Expand All @@ -117,4 +121,4 @@ path = "tests/verify-zero-copy.rs"

[package.metadata.cargo-all-features]
# Disable check-all-features, as the bin feature is purely additive.
skip_feature_sets = [[]]
skip_feature_sets = [["use_icu4c"], ["use_wasm"]]
110 changes: 98 additions & 12 deletions provider/datagen/src/databake.rs → provider/datagen/src/baked_exporter.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,61 @@
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

use databake::{quote, CrateEnv, TokenStream};
//! A data exporter that bakes the data into Rust code.
//!
//! This module can be used as a target for the `icu_datagen` crate.
//!
//! # Examples
//!
//! ```
//! use icu_datagen::prelude::*;
//! use icu_provider::hello_world::*;
//! use icu_datagen::baked_exporter::*;
//!
//! let demo_path = std::env::temp_dir().join("icu4x_baked_demo");
//!
//! // Set up the exporter
//! let mut exporter = BakedExporter::new(demo_path.clone(), Default::default()).unwrap();
//!
//! // Export something
//! DatagenProvider::default()
//! .export(
//! [HelloWorldV1Marker::KEY].into_iter().collect(),
//! exporter
//! ).unwrap();
//! #
//! # std::fs::remove_dir_all(&demo_path)
//! # .expect("Should clean up test directory");
//! ```
//!
//! The resulting module structure can now be used like this:
//!
//! ```compile_fail
//! use icu_provider::prelude::*;
//!
//! struct MyDataProvider;
//!
//! mod baked {
//! include!("/path/to/mod/")
//! impl_data_provider!(MyDataProvider);
//! }
//!
//! let response: DataPayload<HelloWorldV1Marker> = provider
//! .load(Default::default())
//! .unwrap()
//! .take_payload()
//! .unwrap();
//! ```

use databake::*;
use icu_provider::datagen::*;
use icu_provider::prelude::*;
use rayon::prelude::*;
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::collections::HashMap;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use std::sync::Mutex;

use crate::BakedOptions;

macro_rules! move_out {
($field:expr) => {{
let mut tmp = Default::default();
Expand All @@ -27,8 +68,37 @@ macro_rules! move_out {
// TokenStream isn't Send/Sync
type SyncTokenStream = String;

/// Options for configuring the output of [`BakedExporter`].
#[non_exhaustive]
#[derive(Debug)]
pub struct Options {
/// Whether to run `rustfmt` on the generated files.
pub pretty: bool,
/// Whether to gate each key on its crate name. This allows using the module
/// even if some keys are not required and their dependencies are not included.
/// Requires use_separate_crates.
pub insert_feature_gates: bool,
/// Whether to use separate crates to name types instead of the `icu` metacrate
pub use_separate_crates: bool,
/// Whether to overwrite existing data. By default, errors if it is present.
pub overwrite: bool,
}

#[allow(clippy::derivable_impls)] // want to be explicit about bool defaults
impl Default for Options {
fn default() -> Self {
Self {
pretty: false,
insert_feature_gates: false,
use_separate_crates: false,
overwrite: false,
}
}
}

#[allow(clippy::type_complexity)]
pub(crate) struct BakedDataExporter {
/// See the module-level documentation for details.
pub struct BakedExporter {
// Input arguments
mod_directory: PathBuf,
pretty: bool,
Expand All @@ -54,9 +124,22 @@ struct ImplData {
feature: SyncTokenStream,
}

impl BakedDataExporter {
pub fn new(mod_directory: PathBuf, options: BakedOptions) -> Result<Self, DataError> {
let BakedOptions {
impl std::fmt::Debug for BakedExporter {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BakedExporter")
.field("mod_directory", &self.mod_directory)
.field("pretty", &self.pretty)
.field("insert_feature_gates", &self.insert_feature_gates)
.field("use_separate_crates", &self.use_separate_crates)
// skip formatting intermediate data
.finish()
}
}

impl BakedExporter {
/// Constructs a new [`BakedExporter`] with the given output directory and options.
pub fn new(mod_directory: PathBuf, options: Options) -> Result<Self, DataError> {
let Options {
pretty,
insert_feature_gates,
use_separate_crates,
Expand Down Expand Up @@ -179,6 +262,7 @@ impl BakedDataExporter {
}

fn write_intermediate_mod_files(&mut self) -> Result<(), DataError> {
use rayon::prelude::*;
move_out!(self.mod_files)
.into_inner()
.expect("poison")
Expand All @@ -199,7 +283,7 @@ impl BakedDataExporter {
}
}

impl DataExporter for BakedDataExporter {
impl DataExporter for BakedExporter {
fn put_payload(
&self,
key: DataKey,
Expand Down Expand Up @@ -377,6 +461,8 @@ impl DataExporter for BakedDataExporter {
}

fn close(&mut self) -> Result<(), DataError> {
log::info!("Writing module structure...");

// These are BTreeMaps keyed on the marker to keep the output sorted and stable
let mut data_impls = BTreeMap::new();
let mut any_consts = BTreeMap::new();
Expand Down
Loading

0 comments on commit 115554b

Please sign in to comment.