From e5d57fc78b0098c27aabdd26611cad83c9d92d2c Mon Sep 17 00:00:00 2001 From: Robert Bastian Date: Wed, 4 Jan 2023 16:03:12 +0100 Subject: [PATCH] iterate --- Cargo.lock | 1 + ffi/capi_cdylib/Cargo.toml | 3 +- ffi/capi_staticlib/Cargo.toml | 3 +- ffi/diplomat/Cargo.toml | 15 +- ffi/diplomat/build.rs | 34 +++ .../c/examples/fixeddecimal_tiny/Makefile | 18 +- ffi/diplomat/cpp/docs/source/provider_ffi.rst | 2 +- .../cpp/include/ICU4XDataProvider.hpp | 4 +- ffi/diplomat/js/docs/source/provider_ffi.rst | 2 +- .../js/include/ICU4XDataProvider.d.ts | 2 +- ffi/diplomat/src/provider.rs | 262 ++++-------------- provider/datagen/src/databake.rs | 220 ++++++++------- provider/testdata/data/baked/mod.rs | 100 +++++++ 13 files changed, 331 insertions(+), 335 deletions(-) create mode 100644 ffi/diplomat/build.rs diff --git a/Cargo.lock b/Cargo.lock index c5bfefe8d13..97380c51112 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1384,6 +1384,7 @@ dependencies = [ "icu_calendar", "icu_collator", "icu_collections", + "icu_datagen", "icu_datetime", "icu_decimal", "icu_list", diff --git a/ffi/capi_cdylib/Cargo.toml b/ffi/capi_cdylib/Cargo.toml index 43982d75aed..59c1ed5e692 100644 --- a/ffi/capi_cdylib/Cargo.toml +++ b/ffi/capi_cdylib/Cargo.toml @@ -39,9 +39,8 @@ icu_provider = { version = "1.0.0", path = "../../provider/core", default-featur # Please keep features/cargo-all-features lists in sync with the icu_capi crate [features] default = ["icu_capi/default"] -any_provider = ["icu_capi/any_provider"] -buffer_provider = ["icu_capi/buffer_provider"] baked_provider = ["icu_capi/baked_provider"] +buffer_provider = ["icu_capi/buffer_provider"] provider_fs = ["icu_capi/provider_fs"] # Indirectly implies buffer_provider provider_test = ["icu_capi/provider_test"] logging = ["icu_capi/logging"] diff --git a/ffi/capi_staticlib/Cargo.toml b/ffi/capi_staticlib/Cargo.toml index 2782db11d0b..f0ede141bae 100644 --- a/ffi/capi_staticlib/Cargo.toml +++ b/ffi/capi_staticlib/Cargo.toml @@ -39,9 +39,8 @@ icu_provider = { version = "1.0.0", path = "../../provider/core", default-featur # Please keep features/cargo-all-features lists in sync with the icu_capi crate [features] default = ["icu_capi/default"] -any_provider = ["icu_capi/any_provider"] -buffer_provider = ["icu_capi/buffer_provider"] baked_provider = ["icu_capi/baked_provider"] +buffer_provider = ["icu_capi/buffer_provider"] provider_fs = ["icu_capi/provider_fs"] # Indirectly implies buffer_provider provider_test = ["icu_capi/provider_test"] logging = ["icu_capi/logging"] diff --git a/ffi/diplomat/Cargo.toml b/ffi/diplomat/Cargo.toml index a5d4fe0660c..79c4cd4f2c0 100644 --- a/ffi/diplomat/Cargo.toml +++ b/ffi/diplomat/Cargo.toml @@ -32,13 +32,12 @@ all-features = true skip_optional_dependencies = true # Bench feature gets tested separately and is only relevant for CI. # logging enables a feature of a dependency that has no externally visible API changes -# baked_provider requires datagen. -denylist = ["bench", "logging", "baked_provider"] +denylist = ["bench", "logging"] # Please keep the features list in sync with the icu_capi_staticlib/icu_capi_cdylib crates [features] -default = ["any_provider"] -any_provider = [] +default = ["baked_provider"] +baked_provider = ["dep:zerovec"] buffer_provider = [ "dep:icu_provider_blob", "dep:serde", @@ -54,14 +53,12 @@ buffer_provider = [ "icu_provider/serde", "icu_provider_adapters/serde", "icu_segmenter/serde", - "icu_testdata?/buffer", ] provider_fs = ["dep:icu_provider_fs", "buffer_provider"] -provider_test = ["dep:icu_testdata"] +provider_test = ["dep:icu_testdata", "buffer_provider"] logging = ["icu_provider/log_error_context", "dep:log"] # Use the env_logger functionality to log based on environment variables simple_logger = ["dep:simple_logger"] -baked_provider = ["any_provider", "dep:zerovec"] # meta feature for things we enable by default in C and C++ cpp_default = ["provider_test", "logging", "simple_logger"] @@ -93,7 +90,7 @@ writeable = { version = "0.5", path = "../../utils/writeable/" } icu_provider_blob = { version = "1.0.0", path = "../../provider/blob", optional = true } serde = { version = "1.0", default-features = false, optional = true } -icu_testdata = { version = "1.0.0", path = "../../provider/testdata", optional = true, features = ["icu_segmenter"] } +icu_testdata = { version = "1.0.0", path = "../../provider/testdata", optional = true, features = ["buffer"] } zerovec = { version = "*", path = "../../utils/zerovec", optional = true} @@ -104,6 +101,8 @@ zerovec = { version = "*", path = "../../utils/zerovec", optional = true} diplomat = { git = "https://github.com/rust-diplomat/diplomat", rev = "d3010a4b88a0f37d23e3c64cf5500cedc4e8cdf0" } diplomat-runtime = { git = "https://github.com/rust-diplomat/diplomat", rev = "d3010a4b88a0f37d23e3c64cf5500cedc4e8cdf0" } +[build-dependencies] +icu_datagen = { version = "1.0", path = "../../provider/datagen", features = ["icu_segmenter"]} [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Logging is automagical in wasm, we only need this for native diff --git a/ffi/diplomat/build.rs b/ffi/diplomat/build.rs new file mode 100644 index 00000000000..3d79c39fc24 --- /dev/null +++ b/ffi/diplomat/build.rs @@ -0,0 +1,34 @@ +// This file is part of ICU4X. For terms of use, please see the file +// called LICENSE at the top level of the ICU4X source tree +// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ). + +use icu_datagen::*; +use std::path::PathBuf; + +fn main() { + if std::env::var("CARGO_FEATURE_BAKED_PROVIDER").is_ok() + && std::env::var("ICU4X_FFI_BAKED_ROOT").is_err() + { + let mod_directory = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()).join("empty_bake"); + println!( + "cargo:rustc-env=ICU4X_FFI_BAKED_ROOT={}", + mod_directory.display() + ); + + if !mod_directory.exists() { + icu_datagen::datagen( + Some(&[]), + &[], + &SourceData::default(), + vec![Out::Module { + mod_directory, + insert_feature_gates: false, + use_separate_crates: true, + overwrite: false, + pretty: false, + }], + ) + .unwrap(); + } + } +} diff --git a/ffi/diplomat/c/examples/fixeddecimal_tiny/Makefile b/ffi/diplomat/c/examples/fixeddecimal_tiny/Makefile index 2b5b23ed183..fc287980959 100644 --- a/ffi/diplomat/c/examples/fixeddecimal_tiny/Makefile +++ b/ffi/diplomat/c/examples/fixeddecimal_tiny/Makefile @@ -17,21 +17,21 @@ GCC := gcc CLANG := clang-14 LLD := lld-14 -ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) +export ICU4X_FFI_BAKED_ROOT = $(dir $(realpath $(lastword $(MAKEFILE_LIST))))baked -../../../../../target/debug/libicu_capi_staticlib.a: $(ALL_RUST) baked/mod.rs - ICU4X_FFI_BAKED_ROOT=$(ROOT_DIR)/baked/mod.rs cargo build -p icu_capi_staticlib --no-default-features --features baked_provider +../../../../../target/debug/libicu_capi_staticlib.a: $(ALL_RUST) $(ICU4X_FFI_BAKED_ROOT) + cargo build -p icu_capi_staticlib --no-default-features --features baked_provider -icu_capi_staticlib_tiny/target/x86_64-unknown-linux-gnu/debug/libicu_capi_staticlib_tiny.a: $(ALL_RUST) baked/mod.rs +icu_capi_staticlib_tiny/target/x86_64-unknown-linux-gnu/debug/libicu_capi_staticlib_tiny.a: $(ALL_RUST) $(ICU4X_FFI_BAKED_ROOT) cd icu_capi_staticlib_tiny && \ - ICU4X_FFI_BAKED_ROOT=$(ROOT_DIR)/baked/mod.rs RUSTFLAGS="-Clinker-plugin-lto -Clinker=$(CLANG) -Ccodegen-units=1 -Clink-arg=-flto -Cpanic=abort" cargo +${ICU4X_NIGHTLY_TOOLCHAIN} panic-abort-build --target x86_64-unknown-linux-gnu + RUSTFLAGS="-Clinker-plugin-lto -Clinker=$(CLANG) -Ccodegen-units=1 -Clink-arg=-flto -Cpanic=abort" cargo +$(ICU4X_NIGHTLY_TOOLCHAIN) panic-abort-build --target x86_64-unknown-linux-gnu -icu_capi_staticlib_tiny/target/x86_64-unknown-linux-gnu/release-opt-size/libicu_capi_staticlib_tiny.a: $(ALL_RUST) baked/mod.rs +icu_capi_staticlib_tiny/target/x86_64-unknown-linux-gnu/release-opt-size/libicu_capi_staticlib_tiny.a: $(ALL_RUST) $(ICU4X_FFI_BAKED_ROOT) cd icu_capi_staticlib_tiny && \ - ICU4X_FFI_BAKED_ROOT=$(ROOT_DIR)/baked/mod.rs RUSTFLAGS="-Clinker-plugin-lto -Clinker=$(CLANG) -Ccodegen-units=1 -Clink-arg=-flto -Cpanic=abort" cargo +${ICU4X_NIGHTLY_TOOLCHAIN} panic-abort-build --target x86_64-unknown-linux-gnu --profile=release-opt-size + RUSTFLAGS="-Clinker-plugin-lto -Clinker=$(CLANG) -Ccodegen-units=1 -Clink-arg=-flto -Cpanic=abort" cargo +$(ICU4X_NIGHTLY_TOOLCHAIN) panic-abort-build --target x86_64-unknown-linux-gnu --profile=release-opt-size -baked/mod.rs: - cargo run -p icu_datagen --features bin,icu_segmenter -- --locales en bn --all-keys --cldr-tag 42.0.0 --icuexport-tag release-72-1 --format mod --out baked --use-separate-crates +$(ICU4X_FFI_BAKED_ROOT): + cargo run -p icu_datagen --features bin,icu_segmenter -- --locales en bn --keys "decimal/symbols@1" --cldr-root ../../../../../provider/testdata/data/cldr/ --format mod --use-separate-crates --out ${ICU4X_FFI_BAKED_ROOT} # Naive target: no optimizations, full std optim0.elf: ../../../../../target/debug/libicu_capi_staticlib.a $(ALL_HEADERS) test.c diff --git a/ffi/diplomat/cpp/docs/source/provider_ffi.rst b/ffi/diplomat/cpp/docs/source/provider_ffi.rst index d59810beb25..f349316ea5a 100644 --- a/ffi/diplomat/cpp/docs/source/provider_ffi.rst +++ b/ffi/diplomat/cpp/docs/source/provider_ffi.rst @@ -42,7 +42,7 @@ Constructs a :cpp:class:`ICU4XDataProvider` containing baked data. - When compiling the Rust library, set the ``ICU4X_FFI_BAKED_ROOT`` environment variable to the baked data's ``mod.rs``'s path. + When compiling the Rust library, set the ``ICU4X_FFI_BAKED_ROOT`` environment variable to the baked data folder. .. cpp:function:: diplomat::result fork_by_key(ICU4XDataProvider& other) diff --git a/ffi/diplomat/cpp/include/ICU4XDataProvider.hpp b/ffi/diplomat/cpp/include/ICU4XDataProvider.hpp index fbd086955a0..52b631554d7 100644 --- a/ffi/diplomat/cpp/include/ICU4XDataProvider.hpp +++ b/ffi/diplomat/cpp/include/ICU4XDataProvider.hpp @@ -68,8 +68,8 @@ class ICU4XDataProvider { /** * Constructs a [`ICU4XDataProvider`] containing baked data. * - * When compiling the Rust library, set the `ICU4X_FFI_BAKED_ROOT` environment - * variable to the baked data's `mod.rs`'s path. + * When compiling the Rust library, set the `ICU4X_FFI_BAKED_ROOT` + * environment variable to the baked data folder. */ static ICU4XDataProvider create_baked(); diff --git a/ffi/diplomat/js/docs/source/provider_ffi.rst b/ffi/diplomat/js/docs/source/provider_ffi.rst index a98740e020b..6d139ec3cd4 100644 --- a/ffi/diplomat/js/docs/source/provider_ffi.rst +++ b/ffi/diplomat/js/docs/source/provider_ffi.rst @@ -44,7 +44,7 @@ Constructs a :js:class:`ICU4XDataProvider` containing baked data. - When compiling the Rust library, set the ``ICU4X_FFI_BAKED_ROOT`` environment variable to the baked data's ``mod.rs``'s path. + When compiling the Rust library, set the ``ICU4X_FFI_BAKED_ROOT`` environment variable to the baked data folder. .. js:function:: fork_by_key(other) diff --git a/ffi/diplomat/js/include/ICU4XDataProvider.d.ts b/ffi/diplomat/js/include/ICU4XDataProvider.d.ts index 45ac699dee3..8e9124dd56b 100644 --- a/ffi/diplomat/js/include/ICU4XDataProvider.d.ts +++ b/ffi/diplomat/js/include/ICU4XDataProvider.d.ts @@ -48,7 +48,7 @@ export class ICU4XDataProvider { * Constructs a {@link ICU4XDataProvider `ICU4XDataProvider`} containing baked data. - * When compiling the Rust library, set the `ICU4X_FFI_BAKED_ROOT` environment variable to the baked data's `mod.rs`'s path. + * When compiling the Rust library, set the `ICU4X_FFI_BAKED_ROOT` environment variable to the baked data folder. */ static create_baked(): ICU4XDataProvider; diff --git a/ffi/diplomat/src/provider.rs b/ffi/diplomat/src/provider.rs index 674fb085dfd..f89ba9d646c 100644 --- a/ffi/diplomat/src/provider.rs +++ b/ffi/diplomat/src/provider.rs @@ -7,7 +7,10 @@ use alloc::boxed::Box; use icu_provider::prelude::*; #[allow(unused_imports)] // feature-specific use icu_provider::MaybeSendSync; -use icu_provider_adapters::empty::EmptyDataProvider; +#[allow(unused_imports)] // feature-specific +use icu_provider_adapters::{ + empty::EmptyDataProvider, fallback::LocaleFallbackProvider, fallback::LocaleFallbacker, +}; #[allow(unused_imports)] // feature-specific use yoke::{trait_hack::YokeTraitHack, Yokeable}; #[allow(unused_imports)] // feature-specific @@ -15,10 +18,8 @@ use zerofrom::ZeroFrom; pub enum ICU4XDataProviderInner { Empty, - #[cfg(feature = "any_provider")] - Any(Box), #[cfg(feature = "baked_provider")] - Baked, + Baked(Option>), #[cfg(feature = "buffer_provider")] Buffer(Box), } @@ -33,41 +34,27 @@ impl Default for ICU4XDataProviderInner { struct BakedProvider; #[cfg(feature = "baked_provider")] mod baked { - include!(core::env!("ICU4X_FFI_BAKED_ROOT")); - impl_data_provider!(super::BakedProvider); - #[cfg(feature = "any_provider")] - impl_any_provider!(super::BakedProvider); + include!(concat!(core::env!("ICU4X_FFI_BAKED_ROOT"), "/mod.rs")); + impl_data_provider!(super::BakedProvider, COMPLETE); } #[diplomat::bridge] pub mod ffi { use super::ICU4XDataProviderInner; - #[cfg(feature = "baked_provider")] - use super::BakedProvider; use crate::errors::ffi::ICU4XError; use crate::fallbacker::ffi::ICU4XLocaleFallbacker; use alloc::boxed::Box; use diplomat_runtime::DiplomatResult; #[allow(unused_imports)] // feature-gated - use icu_provider_adapters::fallback::LocaleFallbackProvider; - #[allow(unused_imports)] // feature-gated use icu_provider_adapters::fork::predicates::MissingLocalePredicate; + #[allow(unused_imports)] // feature-gated + use icu_provider_adapters::{fallback::LocaleFallbackProvider, fallback::LocaleFallbacker}; #[diplomat::opaque] /// An ICU4X data provider, capable of loading ICU4X data keys from some source. #[diplomat::rust_link(icu_provider, Mod)] pub struct ICU4XDataProvider(pub ICU4XDataProviderInner); - #[cfg(feature = "any_provider")] - #[allow(dead_code)] // feature-specific - fn convert_any_provider( - x: D, - ) -> Box { - Box::new(ICU4XDataProvider(super::ICU4XDataProviderInner::Any( - Box::new(x), - ))) - } - #[cfg(feature = "buffer_provider")] fn convert_buffer_provider( x: D, @@ -115,20 +102,7 @@ pub mod ffi { #[cfg(not(feature = "provider_test"))] panic!("Requires feature 'provider_test'"); - #[cfg(all( - feature = "provider_test", - not(any(feature = "any_provider", feature = "buffer_provider")) - ))] - panic!("Requires feature 'any_provider' or 'buffer_provider'"); - - #[cfg(all(feature = "provider_test", feature = "any_provider"))] - return convert_any_provider(icu_testdata::any()); - - #[cfg(all( - feature = "provider_test", - feature = "buffer_provider", - not(feature = "any_provider") - ))] + #[cfg(feature = "provider_test")] return convert_buffer_provider(icu_testdata::buffer()); } @@ -161,14 +135,14 @@ pub mod ffi { /// Constructs a [`ICU4XDataProvider`] containing baked data. /// - /// When compiling the Rust library, set the `ICU4X_FFI_BAKED_ROOT` environment - /// variable to the baked data's `mod.rs`'s path. + /// When compiling the Rust library, set the `ICU4X_FFI_BAKED_ROOT` + /// environment variable to the baked data folder. pub fn create_baked() -> Box { #[cfg(not(feature = "baked_provider"))] panic!("Requires feature 'baked_provider'"); #[cfg(feature = "baked_provider")] - Box::new(ICU4XDataProvider(ICU4XDataProviderInner::Baked)) + Box::new(ICU4XDataProvider(ICU4XDataProviderInner::Baked(None))) } /// Creates a provider that tries the current provider and then, if the current provider @@ -192,19 +166,14 @@ pub mod ffi { let a = core::mem::take(&mut self.0); let b = core::mem::take(&mut other.0); match (a, b) { - #[cfg(feature = "any_provider")] - (ICU4XDataProviderInner::Any(a), ICU4XDataProviderInner::Any(b)) => { - self.0 = ICU4XDataProviderInner::Any(Box::from( - icu_provider_adapters::fork::ForkByKeyProvider::new(a, b), - )); - Ok(()) - } - #[cfg(all(feature = "any_provider", feature = "baked_provider"))] - (ICU4XDataProviderInner::Baked, ICU4XDataProviderInner::Any(a)) - | (ICU4XDataProviderInner::Any(a), ICU4XDataProviderInner::Baked) => { - self.0 = ICU4XDataProviderInner::Any(Box::from( - icu_provider_adapters::fork::ForkByKeyProvider::new(a, BakedProvider), - )); + #[cfg(feature = "baked_provider")] + // Baked data providers are complete, so fork_by_key results in the first provider + (a @ ICU4XDataProviderInner::Baked(_), ICU4XDataProviderInner::Baked(_)) => { + self.0 = a; + #[cfg(feature = "logging")] + log::warn!( + "Forking two baked providers by key is a noop, as baked providers contain all keys", + ); Ok(()) } #[cfg(feature = "buffer_provider")] @@ -238,27 +207,16 @@ pub mod ffi { let a = core::mem::take(&mut self.0); let b = core::mem::take(&mut other.0); match (a, b) { - #[cfg(feature = "any_provider")] - (ICU4XDataProviderInner::Any(a), ICU4XDataProviderInner::Any(b)) => { - self.0 = ICU4XDataProviderInner::Any(Box::from( - icu_provider_adapters::fork::ForkByErrorProvider::new_with_predicate( - a, - b, - MissingLocalePredicate, - ), - )); - Ok(()) - } - #[cfg(all(feature = "any_provider", feature = "baked_provider"))] - (ICU4XDataProviderInner::Baked, ICU4XDataProviderInner::Any(a)) - | (ICU4XDataProviderInner::Any(a), ICU4XDataProviderInner::Baked) => { - self.0 = ICU4XDataProviderInner::Any(Box::from( - icu_provider_adapters::fork::ForkByErrorProvider::new_with_predicate( - a, - BakedProvider, - MissingLocalePredicate, - ), - )); + #[cfg(feature = "baked_provider")] + (a @ ICU4XDataProviderInner::Baked(_), ICU4XDataProviderInner::Baked(_)) => { + // The "logical" behaviour here would be to try all fallbacks in order + // (i.e `fallbacker: Vec>`). Let's not though. + #[cfg(feature = "logging")] + log::warn!( + "Forking two baked providers by locale is a noop, as the only way they + can differ is by fallback strategy, which results in confusing behaviour.", + ); + self.0 = a; Ok(()) } #[cfg(feature = "buffer_provider")] @@ -301,21 +259,11 @@ pub mod ffi { ICU4XDataProviderInner::Empty => Err(icu_provider::DataErrorKind::MissingDataKey .into_error() .into()), - #[cfg(feature = "any_provider")] - ICU4XDataProviderInner::Any(inner) => { - match LocaleFallbackProvider::try_new_with_any_provider(inner) { + #[cfg(feature = "baked_provider")] + ICU4XDataProviderInner::Baked(_) => { + match LocaleFallbacker::try_new_unstable(&super::BakedProvider) { Ok(x) => { - self.0 = ICU4XDataProviderInner::Any(Box::new(x)); - Ok(()) - } - Err(e) => Err(e.into()), - } - } - #[cfg(all(feature = "any_provider", feature = "baked_provider"))] - ICU4XDataProviderInner::Baked => { - match LocaleFallbackProvider::try_new_with_any_provider(BakedProvider) { - Ok(x) => { - self.0 = ICU4XDataProviderInner::Any(Box::new(x)); + self.0 = ICU4XDataProviderInner::Baked(Some(Box::new(x))); Ok(()) } Err(e) => Err(e.into()), @@ -353,18 +301,9 @@ pub mod ffi { ICU4XDataProviderInner::Empty => Err(icu_provider::DataErrorKind::MissingDataKey .into_error() .into()), - #[cfg(feature = "any_provider")] - ICU4XDataProviderInner::Any(inner) => { - self.0 = ICU4XDataProviderInner::Any(Box::new( - LocaleFallbackProvider::new_with_fallbacker(inner, fallbacker.0.clone()), - )); - Ok(()) - } - #[cfg(all(feature = "any_provider", feature = "baked_provider"))] - ICU4XDataProviderInner::Baked => { - self.0 = ICU4XDataProviderInner::Any(Box::new( - LocaleFallbackProvider::new_with_fallbacker(BakedProvider, fallbacker.0.clone()), - )); + #[cfg(feature = "baked_provider")] + ICU4XDataProviderInner::Baked(_) => { + self.0 = ICU4XDataProviderInner::Baked(Some(Box::new(fallbacker.0.clone()))); Ok(()) } #[cfg(feature = "buffer_provider")] @@ -380,11 +319,7 @@ pub mod ffi { } } -#[cfg(not(any( - feature = "any_provider", - feature = "baked_provider", - feature = "buffer_provider" -)))] +#[cfg(not(any(feature = "baked_provider", feature = "buffer_provider")))] impl DataProvider for ICU4XDataProviderInner where M: KeyedDataMarker + 'static, @@ -394,11 +329,7 @@ where } } -#[cfg(all( - feature = "buffer_provider", - not(feature = "any_provider"), - not(feature = "baked_provider") -))] +#[cfg(all(feature = "buffer_provider", not(feature = "baked_provider")))] impl DataProvider for ICU4XDataProviderInner where M: KeyedDataMarker + 'static, @@ -410,8 +341,6 @@ where fn load(&self, req: DataRequest) -> Result, DataError> { match self { ICU4XDataProviderInner::Empty => EmptyDataProvider::new().load(req), - #[cfg(feature = "baked_provider")] - ICU4XDataProviderInner::Baked => BakedProvider.load(req), ICU4XDataProviderInner::Buffer(buffer_provider) => { buffer_provider.as_deserializing().load(req) } @@ -419,65 +348,19 @@ where } } -#[cfg(all( - feature = "any_provider", - not(feature = "buffer_provider"), - not(feature = "baked_provider") -))] +#[cfg(all(feature = "baked_provider", not(feature = "buffer_provider"),))] impl DataProvider for ICU4XDataProviderInner where M: KeyedDataMarker + 'static, - for<'a> YokeTraitHack<>::Output>: Clone, - M::Yokeable: ZeroFrom<'static, M::Yokeable>, - M::Yokeable: MaybeSendSync, -{ - fn load(&self, req: DataRequest) -> Result, DataError> { - match self { - ICU4XDataProviderInner::Empty => EmptyDataProvider::new().load(req), - ICU4XDataProviderInner::Any(any_provider) => any_provider.as_downcasting().load(req), - } - } -} - -#[cfg(all( - feature = "baked_provider", - not(feature = "buffer_provider"), - not(feature = "any_provider") -))] -impl DataProvider for ICU4XDataProviderInner -where BakedProvider: DataProvider, { fn load(&self, req: DataRequest) -> Result, DataError> { match self { ICU4XDataProviderInner::Empty => EmptyDataProvider::new().load(req), - ICU4XDataProviderInner::Baked => BakedProvider.load(req), - } - } -} - -#[cfg(all( - feature = "buffer_provider", - feature = "any_provider", - not(feature = "baked_provider") -))] -impl DataProvider for ICU4XDataProviderInner -where - M: KeyedDataMarker + 'static, - for<'a> YokeTraitHack<>::Output>: Clone, - M::Yokeable: ZeroFrom<'static, M::Yokeable>, - M::Yokeable: MaybeSendSync, - // Actual bound: - // for<'de> >::Output: Deserialize<'de>, - // Necessary workaround bound (see `yoke::trait_hack` docs): - for<'de> YokeTraitHack<>::Output>: serde::Deserialize<'de>, -{ - fn load(&self, req: DataRequest) -> Result, DataError> { - match self { - ICU4XDataProviderInner::Empty => EmptyDataProvider::new().load(req), - ICU4XDataProviderInner::Any(any_provider) => any_provider.as_downcasting().load(req), - ICU4XDataProviderInner::Buffer(buffer_provider) => { - buffer_provider.as_deserializing().load(req) + ICU4XDataProviderInner::Baked(None) => BakedProvider.load(req), + ICU4XDataProviderInner::Baked(Some(fallbacker)) => { + LocaleFallbackProvider::new_with_fallbacker(BakedProvider, *fallbacker.clone()) + .load(req) } } } @@ -503,58 +386,11 @@ where fn load(&self, req: DataRequest) -> Result, DataError> { match self { ICU4XDataProviderInner::Empty => EmptyDataProvider::new().load(req), - ICU4XDataProviderInner::Baked => BakedProvider.load(req), - ICU4XDataProviderInner::Buffer(buffer_provider) => { - buffer_provider.as_deserializing().load(req) + ICU4XDataProviderInner::Baked(None) => BakedProvider.load(req), + ICU4XDataProviderInner::Baked(Some(fallbacker)) => { + LocaleFallbackProvider::new_with_fallbacker(BakedProvider, *fallbacker.clone()) + .load(req) } - } - } -} - -#[cfg(all( - feature = "any_provider", - feature = "baked_provider", - not(feature = "buffer_provider") -))] -impl DataProvider for ICU4XDataProviderInner -where - M: KeyedDataMarker + 'static, - for<'a> YokeTraitHack<>::Output>: Clone, - M::Yokeable: ZeroFrom<'static, M::Yokeable>, - M::Yokeable: MaybeSendSync, - BakedProvider: DataProvider, -{ - fn load(&self, req: DataRequest) -> Result, DataError> { - match self { - ICU4XDataProviderInner::Empty => EmptyDataProvider::new().load(req), - ICU4XDataProviderInner::Baked => BakedProvider.load(req), - ICU4XDataProviderInner::Any(any_provider) => any_provider.as_downcasting().load(req), - } - } -} - -#[cfg(all( - feature = "any_provider", - feature = "baked_provider", - feature = "buffer_provider" -))] -impl DataProvider for ICU4XDataProviderInner -where - M: KeyedDataMarker + 'static, - for<'a> YokeTraitHack<>::Output>: Clone, - M::Yokeable: ZeroFrom<'static, M::Yokeable>, - M::Yokeable: MaybeSendSync, - // Actual bound: - // for<'de> >::Output: Deserialize<'de>, - // Necessary workaround bound (see `yoke::trait_hack` docs): - for<'de> YokeTraitHack<>::Output>: serde::Deserialize<'de>, - BakedProvider: DataProvider, -{ - fn load(&self, req: DataRequest) -> Result, DataError> { - match self { - ICU4XDataProviderInner::Empty => EmptyDataProvider::new().load(req), - ICU4XDataProviderInner::Baked => BakedProvider.load(req), - ICU4XDataProviderInner::Any(any_provider) => any_provider.as_downcasting().load(req), ICU4XDataProviderInner::Buffer(buffer_provider) => { buffer_provider.as_deserializing().load(req) } diff --git a/provider/datagen/src/databake.rs b/provider/datagen/src/databake.rs index f5265363215..f337dbed9c4 100644 --- a/provider/datagen/src/databake.rs +++ b/provider/datagen/src/databake.rs @@ -36,22 +36,12 @@ pub(crate) struct BakedDataExporter { data: Mutex>>>, // All mod.rs files in the module tree. These can only be written after the last flush. mod_files: Mutex>>, - /// Information to generate implementations. This is populated by `flush` and consumed by `close`. - impl_data: Mutex>, + /// Identifiers of the lookup functions for each key. This is populated by `flush` and consumed by `close`. + lookup_idents: Mutex>, // List of dependencies used by baking. dependencies: CrateEnv, } -/// Data required to write the implementations -struct ImplData { - /// The marker of the key - marker: SyncTokenStream, - /// The path to the lookup function for this marker - lookup_ident: SyncTokenStream, - /// The feature gate for the marker - feature: SyncTokenStream, -} - impl BakedDataExporter { pub fn new( mod_directory: PathBuf, @@ -76,7 +66,7 @@ impl BakedDataExporter { use_separate_crates, data: Default::default(), mod_files: Default::default(), - impl_data: Default::default(), + lookup_idents: Default::default(), dependencies: Default::default(), }) } @@ -188,6 +178,30 @@ impl BakedDataExporter { })?; Ok(()) } + + fn feature_gate(&self, marker: &SyncTokenStream, internal: bool) -> TokenStream { + let feature = if !self.insert_feature_gates { + return quote!(); + } else if marker.contains("DateSkeletonPatternsV1Marker") { + "icu_datetime_experimental" + } else { + let feature = marker + .split(" :: ") + .next() + .unwrap() + .strip_prefix(":: ") + .unwrap(); + if feature.starts_with("icu_provider") { + return quote!(); + } + feature + }; + if internal { + quote!(#![cfg(feature = #feature)]) + } else { + quote!(#[cfg(feature = #feature)]) + } + } } impl DataExporter for BakedDataExporter { @@ -210,25 +224,8 @@ impl DataExporter for BakedDataExporter { } fn flush(&self, key: DataKey) -> Result<(), DataError> { - let marker = - syn::parse2::(crate::registry::key_to_marker_bake(key, &self.dependencies)) - .unwrap(); - - let is_datetime_skeletons = - marker.segments.iter().next_back().unwrap().ident == "DateSkeletonPatternsV1Marker"; - - let feature = if !self.insert_feature_gates { - quote!() - } else if is_datetime_skeletons { - quote! { #![cfg(feature = "icu_datetime_experimental")] } - } else { - let feature = marker.segments.iter().next().unwrap().ident.to_string(); - if !feature.starts_with("icu_provider") { - quote! { #![cfg(feature = #feature)] } - } else { - quote!() - } - }; + let marker = crate::registry::key_to_marker_bake(key, &self.dependencies); + let feature = self.feature_gate(&marker.to_string(), true); // Replace non-ident-allowed tokens. This can still fail if a segment starts with // a token that is not allowed in an initial position. @@ -253,7 +250,7 @@ impl DataExporter for BakedDataExporter { path = path.join(level.ident.to_string()); } - let struct_type = if is_datetime_skeletons { + let struct_type = if marker.to_string().contains("DateSkeletonPatternsV1Marker") { quote! { &'static [( &'static [::icu_datetime::fields::Field], @@ -358,11 +355,10 @@ impl DataExporter for BakedDataExporter { false, )?; - self.impl_data.lock().expect("poison").push(ImplData { - marker: quote!(#marker).to_string(), - lookup_ident: quote!(#module_path :: lookup).to_string(), - feature: feature.to_string().replacen("# ! [", "# [", 1), - }); + self.lookup_idents.lock().expect("poison").insert( + quote!(#marker).to_string(), + quote!(#module_path :: lookup).to_string(), + ); Ok(()) } @@ -370,78 +366,104 @@ impl DataExporter for BakedDataExporter { fn close(&mut self) -> Result<(), DataError> { // These are BTreeMaps keyed on the marker to keep the output sorted and stable let mut data_impls = BTreeMap::new(); + let mut non_requested_data_impls = BTreeMap::new(); let mut any_consts = BTreeMap::new(); let mut any_cases = BTreeMap::new(); - for data in move_out!(self.impl_data) - .into_inner() - .expect("poison") + let mut lookup_idents = move_out!(self.lookup_idents).into_inner().expect("poison"); + + for marker in crate::registry::all_keys() .into_iter() + // HelloWorld is the only key not returned by all_keys + .chain(std::iter::once( + icu_provider::hello_world::HelloWorldV1Marker::KEY, + )) + .map(|k| crate::registry::key_to_marker_bake(k, &self.dependencies)) { - let feature = data.feature.parse::().unwrap(); - let marker = data.marker.parse::().unwrap(); - let lookup_ident = data.lookup_ident.parse::().unwrap(); + let marker_str = marker.to_string(); + let feature = self.feature_gate(&marker_str, false); - data_impls.insert(data.marker.clone(), - quote! { - #feature - impl DataProvider<#marker> for $provider { - fn load( - &self, - req: DataRequest, - ) -> Result, DataError> { - #lookup_ident(&req.locale) - .map(zerofrom::ZeroFrom::zero_from) - .map(DataPayload::from_owned) - .map(|payload| { - DataResponse { - metadata: Default::default(), - payload: Some(payload), - } - }) - .ok_or_else(|| DataErrorKind::MissingLocale.with_req(#marker::KEY, req)) + if let Some(lookup_ident) = lookup_idents.remove(&marker_str) { + let lookup_ident = lookup_ident.parse::().unwrap(); + + data_impls.insert(marker_str.clone(), + quote! { + #feature + impl DataProvider<#marker> for $provider { + fn load( + &self, + req: DataRequest, + ) -> Result, DataError> { + #lookup_ident(&req.locale) + .map(zerofrom::ZeroFrom::zero_from) + .map(DataPayload::from_owned) + .map(|payload| { + DataResponse { + metadata: Default::default(), + payload: Some(payload), + } + }) + .ok_or_else(|| DataErrorKind::MissingLocale.with_req(#marker::KEY, req)) + } } - } - }); + }); + + let hash_ident = marker_str + .split(' ') + .next_back() + .unwrap() + .to_ascii_uppercase() + .parse::() + .unwrap(); - let hash_ident = data - .marker - .split(' ') - .next_back() - .unwrap() - .to_ascii_uppercase() - .parse::() - .unwrap(); - any_consts.insert( - data.marker.clone(), - quote! { - #feature - const #hash_ident: ::icu_provider::DataKeyHash = #marker::KEY.hashed(); - }, - ); - any_cases.insert( - data.marker.clone(), - if data.marker - == ":: icu_datetime :: provider :: calendar :: DateSkeletonPatternsV1Marker" - { + any_consts.insert( + marker_str.clone(), quote! { #feature - #hash_ident => { - #lookup_ident(&req.locale) - .map(zerofrom::ZeroFrom::zero_from) - .map(DataPayload::<#marker>::from_owned) - .map(DataPayload::wrap_into_any_payload) + const #hash_ident: ::icu_provider::DataKeyHash = #marker::KEY.hashed(); + }, + ); + any_cases.insert( + marker_str.clone(), + if marker_str + == ":: icu_datetime :: provider :: calendar :: DateSkeletonPatternsV1Marker" + { + quote! { + #feature + #hash_ident => { + #lookup_ident(&req.locale) + .map(zerofrom::ZeroFrom::zero_from) + .map(DataPayload::<#marker>::from_owned) + .map(DataPayload::wrap_into_any_payload) + } } - } - } else { + } else { + quote! { + #feature + #hash_ident => #lookup_ident(&req.locale).map(AnyPayload::from_static_ref), + } + }, + ); + } else { + non_requested_data_impls.insert( + marker_str.clone(), quote! { #feature - #hash_ident => #lookup_ident(&req.locale).map(AnyPayload::from_static_ref), - } - }, - ); + impl DataProvider<#marker> for $provider { + fn load( + &self, + req: DataRequest, + ) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(#marker::KEY, req)) + } + } + }, + ); + } } + assert!(lookup_idents.is_empty()); + let mods = self .mod_files .get_mut() @@ -452,6 +474,7 @@ impl DataExporter for BakedDataExporter { .map(|p| p.parse::().unwrap()); let data_impls = data_impls.values(); + let non_requested_data_impls = non_requested_data_impls.values(); let any_consts = any_consts.values(); let any_cases = any_cases.values(); @@ -480,7 +503,11 @@ impl DataExporter for BakedDataExporter { macro_rules! impl_data_provider { ($provider:path) => { #(#data_impls)* - } + }; + ($provider:path, COMPLETE) => { + impl_data_provider!($provider); + #(#non_requested_data_impls)* + }; } /// Implement [`AnyProvider`] on the given struct using the data @@ -516,6 +543,7 @@ impl DataExporter for BakedDataExporter { } } + #[allow(dead_code)] struct BakedDataProvider; impl_data_provider!(BakedDataProvider); }, diff --git a/provider/testdata/data/baked/mod.rs b/provider/testdata/data/baked/mod.rs index 57a70ff8727..2b3aae99b04 100644 --- a/provider/testdata/data/baked/mod.rs +++ b/provider/testdata/data/baked/mod.rs @@ -2197,6 +2197,105 @@ macro_rules! impl_data_provider { } } }; + ($ provider : path , COMPLETE) => { + impl_data_provider!($provider); + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::AlnumV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::AlnumV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::BlankV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::BlankV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::CaseSensitiveV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::CaseSensitiveV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::ChangesWhenCasemappedV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::ChangesWhenCasemappedV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::FullCompositionExclusionV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::FullCompositionExclusionV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::GraphV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::GraphV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::GraphemeLinkV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::GraphemeLinkV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::HyphenV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::HyphenV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::NfcInertV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::NfcInertV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::NfdInertV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::NfdInertV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::NfkcInertV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::NfkcInertV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::NfkdInertV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::NfkdInertV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::PrependedConcatenationMarkV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::PrependedConcatenationMarkV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::PrintV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::PrintV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::SegmentStarterV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::SegmentStarterV1Marker::KEY, req)) + } + } + #[cfg(feature = "icu_properties")] + impl DataProvider<::icu_properties::provider::XdigitV1Marker> for $provider { + fn load(&self, req: DataRequest) -> Result, DataError> { + Err(DataErrorKind::MissingDataKey.with_req(::icu_properties::provider::XdigitV1Marker::KEY, req)) + } + } + }; } /// Implement [`AnyProvider`] on the given struct using the data /// hardcoded in this module. This allows the struct to be used with @@ -2922,5 +3021,6 @@ macro_rules! impl_any_provider { } }; } +#[allow(dead_code)] struct BakedDataProvider; impl_data_provider!(BakedDataProvider);