Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add script fallback priority; improve language fallback priority #5344

Merged
merged 8 commits into from
Aug 13, 2024
Merged

Add script fallback priority; improve language fallback priority #5344

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6247645
script fallback
robertbastian Aug 8, 2024
c55d72d
normalize script
robertbastian Aug 9, 2024
fc15e42
Merge branch 'main' into script
robertbastian Aug 13, 2024
1896f0b
don't cross script boundaries when removing region
robertbastian Aug 13, 2024
805b15d
no-data tests
robertbastian Aug 13, 2024
442418a
correctly handle -u-sd
robertbastian Aug 13, 2024
c67d60a
as discussed
robertbastian Aug 13, 2024
5177956
skip fully expanded locales
robertbastian Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
194 changes: 172 additions & 22 deletions components/locale/src/fallback/algorithms.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use icu_locale_core::subtags::{Language, Region, Script};
use super::*;

impl<'a> LocaleFallbackerWithConfig<'a> {
pub(crate) fn normalize(&self, locale: &mut DataLocale) {
pub(crate) fn normalize(&self, locale: &mut DataLocale, default_script: &mut Option<Script>) {
let language = locale.language;
// 1. Populate the region (required for region fallback only)
if self.config.priority == LocaleFallbackPriority::Region && locale.region.is_none() {
Expand All @@ -33,8 +33,8 @@ impl<'a> LocaleFallbackerWithConfig<'a> {
}
}
// 2. Remove the script if it is implied by the other subtags
if let Some(script) = locale.script {
let default_script = locale
if locale.script.is_some() || self.config.priority == LocaleFallbackPriority::Script {
*default_script = locale
.region
.and_then(|region| {
self.likely_subtags.language_region.get_copied(&(
Expand All @@ -48,8 +48,9 @@ impl<'a> LocaleFallbackerWithConfig<'a> {
.get_copied(&language.into_tinystr().to_unvalidated())
.map(|(s, _r)| s)
});

if Some(script) == default_script {
if locale.script == *default_script
&& self.config.priority != LocaleFallbackPriority::Script
{
locale.script = None;
}
}
Expand All @@ -62,6 +63,7 @@ impl<'a> LocaleFallbackIteratorInner<'a> {
pub fn step(&mut self, locale: &mut DataLocale) {
match self.config.priority {
LocaleFallbackPriority::Language => self.step_language(locale),
LocaleFallbackPriority::Script => self.step_script(locale),
LocaleFallbackPriority::Region => self.step_region(locale),
// This case should not normally happen, but `LocaleFallbackPriority` is non_exhaustive.
// Make it go directly to `und`.
Expand All @@ -82,8 +84,6 @@ impl<'a> LocaleFallbackIteratorInner<'a> {
self.backup_subdivision = Some(value);
return;
}
// 3. Assert that the locale is a language identifier
debug_assert!(locale.subdivision.is_none());
// 4. Remove variants
if let Some(single_variant) = locale.variant.take() {
self.backup_variant = Some(single_variant);
Expand Down Expand Up @@ -130,8 +130,6 @@ impl<'a> LocaleFallbackIteratorInner<'a> {
self.backup_subdivision = Some(value);
return;
}
// 3. Assert that the locale is a language identifier
debug_assert!(locale.subdivision.is_none());
// 4. Remove variants
if let Some(variant) = locale.variant.take() {
self.backup_variant = Some(variant);
Expand All @@ -149,6 +147,66 @@ impl<'a> LocaleFallbackIteratorInner<'a> {
locale.region = None;
}

fn step_script(&mut self, locale: &mut DataLocale) {
// Remove the subdivision keyword
if let Some(value) = locale.subdivision.take() {
self.backup_subdivision = Some(value);
return;
}
// Remove variants
if let Some(variant) = locale.variant.take() {
self.backup_variant = Some(variant);
return;
}
// Check for parent override
if let Some((language, script, region)) = self.get_explicit_parent(locale) {
locale.language = language;
locale.script = script;
locale.region = region;
self.restore_subdivision_variants(locale);
return;
}
sffc marked this conversation as resolved.
Show resolved Hide resolved
// Remove the region
if let Some(region) = locale.region {
self.backup_region = Some(region);
locale.region = None;
self.restore_subdivision_variants(locale);
return;
}

// Remove the script if we have a language
if !locale.language.is_empty() {
if locale.script.is_some() {
locale.script = None;
if let Some(region) = self.backup_region.take() {
locale.region = Some(region);
self.restore_subdivision_variants(locale);
}
// needed if more fallback is added at the end
#[allow(clippy::needless_return)]
return;
} else {
// 3. Remove the language and apply the maximized script
locale.language = Language::UND;
locale.script = self.max_script;
self.restore_subdivision_variants(locale);
// needed if more fallback is added at the end
#[allow(clippy::needless_return)]
return;
}
}

// note: UTS #35 wants us to apply "other associated scripts" now. ICU4C/J does not do this,
// so we don't either. They would be found here if they are ever needed:
// https://github.com/unicode-cldr/cldr-core/blob/master/supplemental/languageData.json

// 6. Remove script
if locale.script.is_some() {
locale.script = None;
self.restore_subdivision_variants(locale);
}
}

fn restore_subdivision_variants(&mut self, locale: &mut DataLocale) {
if let Some(subdivision) = self.backup_subdivision.take() {
locale.subdivision = Some(subdivision);
Expand Down Expand Up @@ -178,6 +236,7 @@ mod tests {
requires_data: bool,
// Note: The first entry in the chain is the normalized locale
expected_language_chain: &'static [&'static str],
expected_script_chain: &'static [&'static str],
expected_region_chain: &'static [&'static str],
}

Expand All @@ -187,12 +246,20 @@ mod tests {
input: "en-u-sd-usca",
requires_data: false,
expected_language_chain: &["en-u-sd-usca", "en"],
expected_script_chain: &["en-u-sd-usca", "en", "und-u-sd-usca"],
expected_region_chain: &["en-u-sd-usca", "en", "und-u-sd-usca"],
},
TestCase {
input: "en-US-u-sd-usca",
requires_data: false,
expected_language_chain: &["en-US-u-sd-usca", "en-US", "en-u-sd-usca", "en"],
expected_script_chain: &[
"en-US-u-sd-usca",
"en-US",
"en-u-sd-usca",
"en",
"und-u-sd-usca",
sffc marked this conversation as resolved.
Show resolved Hide resolved
],
expected_region_chain: &["en-US-u-sd-usca", "en-US", "und-US-u-sd-usca", "und-US"],
},
TestCase {
Expand All @@ -206,6 +273,16 @@ mod tests {
"en-fonipa",
"en",
],
expected_script_chain: &[
"en-US-fonipa-u-sd-usca",
"en-US-fonipa",
"en-US",
"en-fonipa-u-sd-usca",
"en-fonipa",
"en",
"und-fonipa-u-sd-usca",
"und-fonipa",
],
expected_region_chain: &[
"en-US-fonipa-u-sd-usca",
"en-US-fonipa",
Expand All @@ -219,30 +296,60 @@ mod tests {
input: "en-u-sd-usca",
requires_data: true,
expected_language_chain: &["en-u-sd-usca", "en"],
expected_script_chain: &[
"en-u-sd-usca",
"en",
"und-Latn-u-sd-usca",
"und-Latn",
"und-u-sd-usca",
],
expected_region_chain: &["en-US-u-sd-usca", "en-US", "und-US-u-sd-usca", "und-US"],
},
TestCase {
input: "en-Latn-u-sd-usca",
requires_data: true,
expected_language_chain: &["en-u-sd-usca", "en"],
expected_script_chain: &[
"en-Latn-u-sd-usca",
"en-Latn",
"en",
"und-Latn-u-sd-usca",
"und-Latn",
"und-u-sd-usca",
],
robertbastian marked this conversation as resolved.
Show resolved Hide resolved
expected_region_chain: &["en-US-u-sd-usca", "en-US", "und-US-u-sd-usca", "und-US"],
},
TestCase {
input: "en-Latn-US-u-sd-usca",
requires_data: true,
expected_language_chain: &["en-US-u-sd-usca", "en-US", "en-u-sd-usca", "en"],
expected_script_chain: &[
"en-Latn-US-u-sd-usca",
"en-Latn-US",
"en-Latn-u-sd-usca",
"en-Latn",
"en-US-u-sd-usca",
"en-US",
"en-u-sd-usca",
"en",
"und-Latn-u-sd-usca",
"und-Latn",
"und-u-sd-usca",
],
expected_region_chain: &["en-US-u-sd-usca", "en-US", "und-US-u-sd-usca", "und-US"],
},
TestCase {
input: "sr-ME",
requires_data: true,
expected_language_chain: &["sr-ME", "sr-Latn-ME", "sr-Latn"],
expected_script_chain: &["sr-ME", "sr", "und-Latn"],
expected_region_chain: &["sr-ME", "und-ME"],
},
TestCase {
input: "sr-Latn-ME",
requires_data: true,
expected_language_chain: &["sr-ME", "sr-Latn-ME", "sr-Latn"],
expected_script_chain: &["sr-Latn-ME", "sr-Latn", "sr-ME", "sr", "und-Latn"],
expected_region_chain: &["sr-ME", "und-ME"],
},
TestCase {
Expand All @@ -256,54 +363,79 @@ mod tests {
"sr-Latn-fonipa",
"sr-Latn",
],
expected_script_chain: &[
"sr-ME-fonipa",
"sr-ME",
"sr-fonipa",
"sr",
"und-Latn-fonipa",
"und-Latn",
"und-fonipa",
],
robertbastian marked this conversation as resolved.
Show resolved Hide resolved
expected_region_chain: &["sr-ME-fonipa", "sr-ME", "und-ME-fonipa", "und-ME"],
},
TestCase {
input: "sr-RS",
requires_data: true,
expected_language_chain: &["sr-RS", "sr"],
expected_script_chain: &["sr-RS", "sr", "und-Cyrl"],
robertbastian marked this conversation as resolved.
Show resolved Hide resolved
expected_region_chain: &["sr-RS", "und-RS"],
},
TestCase {
input: "sr-Cyrl-RS",
requires_data: true,
expected_language_chain: &["sr-RS", "sr"],
expected_script_chain: &["sr-Cyrl-RS", "sr-Cyrl", "sr-RS", "sr", "und-Cyrl"],
expected_region_chain: &["sr-RS", "und-RS"],
},
TestCase {
input: "sr-Latn-RS",
requires_data: true,
expected_language_chain: &["sr-Latn-RS", "sr-Latn"],
expected_script_chain: &["sr-Latn-RS", "sr-Latn", "sr-RS", "sr", "und-Latn"],
expected_region_chain: &["sr-Latn-RS", "und-RS"],
},
TestCase {
input: "de-Latn-LI",
requires_data: true,
expected_language_chain: &["de-LI", "de"],
expected_script_chain: &["de-Latn-LI", "de-Latn", "de-LI", "de", "und-Latn"],
expected_region_chain: &["de-LI", "und-LI"],
},
TestCase {
input: "ca-ES-valencia",
requires_data: true,
expected_language_chain: &["ca-ES-valencia", "ca-ES", "ca-valencia", "ca"],
expected_script_chain: &[
"ca-ES-valencia",
"ca-ES",
"ca-valencia",
"ca",
"und-Latn-valencia",
"und-Latn",
"und-valencia",
],
expected_region_chain: &["ca-ES-valencia", "ca-ES", "und-ES-valencia", "und-ES"],
},
TestCase {
input: "es-AR",
requires_data: true,
expected_language_chain: &["es-AR", "es-419", "es"],
expected_script_chain: &["es-AR", "es-419", "es", "und-Latn"],
expected_region_chain: &["es-AR", "und-AR"],
},
TestCase {
input: "hi-IN",
requires_data: true,
expected_language_chain: &["hi-IN", "hi"],
expected_script_chain: &["hi-IN", "hi", "und-Deva"],
expected_region_chain: &["hi-IN", "und-IN"],
},
TestCase {
input: "hi-Latn-IN",
requires_data: true,
expected_language_chain: &["hi-Latn-IN", "hi-Latn", "en-IN", "en-001", "en"],
expected_script_chain: &["hi-Latn-IN", "hi-Latn", "en-IN", "en-001", "en", "und-Latn"],
expected_region_chain: &["hi-Latn-IN", "und-IN"],
},
TestCase {
Expand All @@ -313,26 +445,50 @@ mod tests {
// The fallback algorithm does not visit the language-script bundle when the
// script is the default for the language
expected_language_chain: &["zh-CN", "zh"],
expected_script_chain: &["zh-CN", "zh", "und-Hans"],
expected_region_chain: &["zh-CN", "und-CN"],
},
TestCase {
input: "zh-TW",
requires_data: true,
expected_language_chain: &["zh-TW", "zh-Hant-TW", "zh-Hant"],
expected_script_chain: &["zh-TW", "zh", "und-Hant"],
expected_region_chain: &["zh-TW", "und-TW"],
},
TestCase {
input: "yue-HK",
requires_data: true,
expected_language_chain: &["yue-HK", "yue"],
expected_script_chain: &["yue-HK", "yue", "und-Hant"],
expected_region_chain: &["yue-HK", "und-HK"],
},
TestCase {
input: "yue-HK",
requires_data: true,
expected_language_chain: &["yue-HK", "yue"],
expected_script_chain: &["yue-HK", "yue", "und-Hant"],
expected_region_chain: &["yue-HK", "und-HK"],
// TODO(#3867): script fallback should do zh-Hant or und-Hant as well
},
TestCase {
input: "az-Arab-IR",
requires_data: true,
expected_language_chain: &["az-IR", "az-Arab-IR", "az-Arab"],
expected_script_chain: &["az-Arab-IR", "az-Arab", "az-IR", "az", "und-Arab"],
expected_region_chain: &["az-IR", "und-IR"],
},
TestCase {
input: "az-IR",
requires_data: true,
expected_language_chain: &["az-IR", "az-Arab-IR", "az-Arab"],
expected_script_chain: &["az-IR", "az", "und-Arab"],
expected_region_chain: &["az-IR", "und-IR"],
},
TestCase {
input: "az-Arab",
requires_data: true,
expected_language_chain: &["az-Arab"],
expected_script_chain: &["az-Arab", "az", "und-Arab"],
expected_region_chain: &["az-IR", "und-IR"],
},
];

Expand All @@ -347,6 +503,7 @@ mod tests {
LocaleFallbackPriority::Language,
cas.expected_language_chain,
),
(LocaleFallbackPriority::Script, cas.expected_script_chain),
(LocaleFallbackPriority::Region, cas.expected_region_chain),
] {
let mut config = LocaleFallbackConfig::default();
Expand All @@ -359,22 +516,15 @@ mod tests {
let mut it = fallbacker
.for_config(config)
.fallback_for(cas.input.parse().unwrap());
for &expected in expected_chain {
assert_eq!(
expected,
&*it.get().write_to_string(),
"{:?} ({:?})",
cas.input,
priority
);
let mut actual_chain = Vec::new();
while !it.get().is_und() {
actual_chain.push(it.get().write_to_string().into_owned());
it.step();
}
assert_eq!(
"und",
&*it.get().write_to_string(),
expected_chain, &actual_chain,
"{:?} ({:?})",
cas.input,
priority
cas.input, priority
);
}
}
Expand Down
Loading