From f8af562f5a6b8b9d536f04e170bc96d4a76aa58e Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 21:39:52 +0200 Subject: [PATCH 01/57] added language files for tests --- testing/i18n-basic/en-US/basic.ftl | 3 +++ testing/i18n-basic/es-MX/basic.ftl | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 testing/i18n-basic/en-US/basic.ftl create mode 100644 testing/i18n-basic/es-MX/basic.ftl diff --git a/testing/i18n-basic/en-US/basic.ftl b/testing/i18n-basic/en-US/basic.ftl new file mode 100644 index 000000000..eb94ab9c7 --- /dev/null +++ b/testing/i18n-basic/en-US/basic.ftl @@ -0,0 +1,3 @@ +greeting = Hello, { $name }! +age = You are { $hours } hours old. +test = This is a test \ No newline at end of file diff --git a/testing/i18n-basic/es-MX/basic.ftl b/testing/i18n-basic/es-MX/basic.ftl new file mode 100644 index 000000000..079ac28f5 --- /dev/null +++ b/testing/i18n-basic/es-MX/basic.ftl @@ -0,0 +1,2 @@ +greeting = ¡Hola, { $name }! +age = Tienes { $hours } horas. From e9b1be5c90d09d62ed2b0e4e73a042647d259c5b Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 21:40:40 +0200 Subject: [PATCH 02/57] added htmls files to test translation --- testing/templates/i18n.html | 2 ++ testing/templates/i18n_invalid.html | 2 ++ testing/templates/i18n_no_args.html | 1 + 3 files changed, 5 insertions(+) create mode 100644 testing/templates/i18n.html create mode 100644 testing/templates/i18n_invalid.html create mode 100644 testing/templates/i18n_no_args.html diff --git a/testing/templates/i18n.html b/testing/templates/i18n.html new file mode 100644 index 000000000..d73e64660 --- /dev/null +++ b/testing/templates/i18n.html @@ -0,0 +1,2 @@ +

{{ localize(greeting, name: name) }}

+

{{ localize(age, hours: hours ) }}

\ No newline at end of file diff --git a/testing/templates/i18n_invalid.html b/testing/templates/i18n_invalid.html new file mode 100644 index 000000000..5e1262048 --- /dev/null +++ b/testing/templates/i18n_invalid.html @@ -0,0 +1,2 @@ +

{{ localize(greetingsss, name: name) }}

+

{{ localize(ages) }}

\ No newline at end of file diff --git a/testing/templates/i18n_no_args.html b/testing/templates/i18n_no_args.html new file mode 100644 index 000000000..f72083b06 --- /dev/null +++ b/testing/templates/i18n_no_args.html @@ -0,0 +1 @@ +

{{ localize(test, test: test) }}

\ No newline at end of file From 98779157f3479d58ba115a2295ac19b3c1ebfd1f Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 21:40:53 +0200 Subject: [PATCH 03/57] added tests --- testing/tests/i18n.rs | 81 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 testing/tests/i18n.rs diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs new file mode 100644 index 000000000..f418b6204 --- /dev/null +++ b/testing/tests/i18n.rs @@ -0,0 +1,81 @@ +// TODO +/* +#![cfg(feature = "with-i18n")] +#![allow(unused)] +*/ + +use askama::init_translation; +use askama::Template; + +init_translation! { + pub MyLocalizer { + static_loader_name: LOCALES, + locales: "i18n-basic", + fallback_language: "en-US", + customise: |bundle| bundle.set_use_isolating(false) + } +} + +#[derive(Template)] +#[template(path = "i18n_invalid.html")] +struct UsesI18nInvalid<'a> { + #[localizer] + loc: MyLocalizer, + name: &'a str, +} + +#[derive(Template)] +#[template(path = "i18n.html")] +struct UsesI18n<'a> { + #[localizer] + loc: MyLocalizer, + name: &'a str, + hours: f64, +} +#[derive(Template)] +#[template(path = "i18n_no_args.html")] +struct UsesNoArgsI18n<'a> { + #[localizer] + loc: MyLocalizer, + test: &'a str, +} + +#[test] +fn existing_language() { + let template = UsesI18n { + loc: MyLocalizer::new(unic_langid::langid!("es-MX")), + name: "Hilda", + hours: 300072.3, + }; + assert_eq!( + template.render().unwrap(), + r#"

¡Hola, Hilda!

+

Tienes 300072.3 horas.

"# + ) +} + +#[test] +fn unknown_language() { + let template = UsesI18n { + loc: MyLocalizer::new(unic_langid::langid!("nl-BE")), + name: "Hilda", + hours: 300072.3, + }; + assert_eq!( + template.render().unwrap(), + r#"

Hello, Hilda!

+

You are 300072.3 hours old.

"# + ) +} + +#[test] +fn no_args() { + let template = UsesNoArgsI18n { + loc: MyLocalizer::new(unic_langid::langid!("es-MX")), + test: "" + }; + assert_eq!( + template.render().unwrap(), + r#"

This is a test

"# + ) +} From 2eb0fdf8f77c983eb8f16f2b99c8823d4a6652e4 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 21:43:26 +0200 Subject: [PATCH 04/57] added parser for localize(foo, bar: baz) --- askama_derive/src/parser.rs | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index efcad7369..3ba54d125 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -66,6 +66,7 @@ pub(crate) enum Expr<'a> { Call(Box>, Vec>), RustMacro(&'a str, &'a str), Try(Box>), + Localize(&'a str, Option<&'a str>, Vec<(&'a str, Expr<'a>)>), } impl Expr<'_> { @@ -668,7 +669,28 @@ macro_rules! expr_prec_layer { } } } - +fn localize(i: &str) -> IResult<&str, Expr<'_>> { + let (tail, (_, _, message, attribute, args, _)) = tuple(( + tag("localize"), + ws(tag("(")), + identifier, + opt(tuple((ws(tag(".")), identifier))), + opt(tuple(( + ws(tag(",")), + separated_list0(ws(tag(",")), tuple((identifier, ws(tag(":")), expr_any))), + ))), + ws(tag(")")), + ))(i)?; + Ok(( + tail, + Expr::Localize( + message, + attribute.map(|(_, a)| a), + args.map(|(_, args)| args.into_iter().map(|(k, _, v)| (k, v)).collect()) + .unwrap_or_default(), + ), + )) +} expr_prec_layer!(expr_muldivmod, expr_filtered, "*", "/", "%"); expr_prec_layer!(expr_addsub, expr_muldivmod, "+", "-"); expr_prec_layer!(expr_shifts, expr_addsub, ">>", "<<"); @@ -682,10 +704,13 @@ expr_prec_layer!(expr_or, expr_and, "||"); fn expr_handle_ws(i: &str) -> IResult<&str, Whitespace> { alt((char('-'), char('+'), char('~')))(i).map(|(s, r)| (s, Whitespace::from(r))) } - fn expr_any(i: &str) -> IResult<&str, Expr<'_>> { let range_right = |i| pair(ws(alt((tag("..="), tag("..")))), opt(expr_or))(i); alt(( + map(localize, |expr| match expr { + Expr::Localize(_, _, _) => expr, + _ => panic!("localize failed: {:?}", expr), + }), map(range_right, |(op, right)| { Expr::Range(op, None, right.map(Box::new)) }), From 7d11fd27765898c0a48298c3e44c74bbf97220b2 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 21:43:55 +0200 Subject: [PATCH 05/57] added tests for the localize parser --- askama_derive/src/parser.rs | 41 ++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 3ba54d125..a19a95105 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -1279,7 +1279,46 @@ mod tests { fn test_invalid_block() { super::parse("{% extend \"blah\" %}", &Syntax::default()).unwrap(); } - + #[test] + fn test_parse_localize() { + assert_eq!( + super::parse("{{ localize(a, v: 32 + 7) }}", &Syntax::default()).unwrap(), + vec![Node::Expr( + Ws(None, None), + Expr::Localize( + "a", + None, + vec![( + "v", + Expr::BinOp("+", Expr::NumLit("32").into(), Expr::NumLit("7").into()) + )] + ) + )] + ); + } + #[test] + fn test_parse_nested_localize() { + assert_eq!( + super::parse("{{ localize(a, v: localize(a, 2) ) }}", &Syntax::default()).unwrap(), + vec![Node::Expr( + Ws(None, None), + Expr::Localize( + "a", + None, + vec![( + "v", + Expr::Localize( + "a", + None, + vec![( + "", + Expr::StrLit("2")) + ] + ))] + ) + )] + ); + } #[test] fn test_parse_filter() { use Expr::*; From 833692999245a499d7c82313298966bac08df609 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 21:53:43 +0200 Subject: [PATCH 06/57] added visit_localize, arm: Expr::Localize, field:localized_messages --- askama_derive/src/generator.rs | 63 +++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index 39a538076..a8dc060cc 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -7,7 +7,7 @@ use crate::CompileError; use proc_macro::TokenStream; use quote::{quote, ToTokens}; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::path::{Path, PathBuf}; use std::{cmp, hash, mem, str}; @@ -254,6 +254,8 @@ struct Generator<'a, S: std::hash::BuildHasher> { // If set to `suppress`, the whitespace characters will be removed by default unless `+` is // used. whitespace: WhitespaceHandling, + // Messages used with localize() + localized_messages: BTreeSet, } impl<'a, S: std::hash::BuildHasher> Generator<'a, S> { @@ -275,6 +277,7 @@ impl<'a, S: std::hash::BuildHasher> Generator<'a, S> { buf_writable: vec![], named: 0, whitespace, + localized_messages: BTreeSet::new(), } } @@ -1292,9 +1295,67 @@ impl<'a, S: std::hash::BuildHasher> Generator<'a, S> { Expr::RustMacro(name, args) => self.visit_rust_macro(buf, name, args), Expr::Try(ref expr) => self.visit_try(buf, expr.as_ref())?, Expr::Tuple(ref exprs) => self.visit_tuple(buf, exprs)?, + Expr::Localize(message, attribute, ref args) => { + self.visit_localize(buf, message, attribute, args)? + } }) } + fn visit_localize( + &mut self, + buf: &mut Buffer, + message: &str, + attribute: Option<&str>, + args: &[(&str, Expr<'_>)], + ) -> Result { + /* TODO + if !cfg!(feature = "with-i18n") { + panic!( + "The askama feature 'with-i18n' must be activated to enable calling `localize`." + ); + } + */ + + // TODO + let localizer = self.input.localizer.as_ref().expect( + "A template struct must have a member with the `#[locale]` \ + attribute that implements `askama::Localize` to enable calling the localize() filter", + ); + + let mut message = message.to_string(); + if let Some(attribute) = attribute { + message.push_str("."); + message.push_str(attribute); + } + + assert!( + message.chars().find(|c| *c == '"').is_none(), + "message ids with quotes in them break the generator, please remove" + ); + + self.localized_messages.insert(message.clone()); + + buf.write(&format!( + "self.{}.translate(\"{}\", &std::iter::FromIterator::from_iter(vec![", + localizer.0, message + )); + + for (i, (name, value)) in args.iter().enumerate() { + if i > 0 { + buf.write(", "); + } + buf.write(&format!( + "(\"{}\".to_string(), ({}).into())", + name, + self.visit_expr_root(value)? + )); + } + buf.write("]))"); + + Ok(DisplayWrap::Unwrapped) + } + + fn visit_try( &mut self, buf: &mut Buffer, From 72dda5ee694372c1c2e26d14b36b761654ea29a7 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 21:55:23 +0200 Subject: [PATCH 07/57] added extraction for the locale field and localizer field --- askama_derive/src/input.rs | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index c09f3d03b..b53c9d721 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -18,6 +18,7 @@ pub(crate) struct TemplateInput<'a> { pub(crate) mime_type: String, pub(crate) parent: Option<&'a syn::Type>, pub(crate) path: PathBuf, + pub(crate) localizer: Option<(syn::Ident, &'a syn::Type)>, } impl TemplateInput<'_> { @@ -53,16 +54,38 @@ impl TemplateInput<'_> { // Check to see if a `_parent` field was defined on the context // struct, and store the type for it for use in the code generator. - let parent = match ast.data { + // Also gets the localizer if there is one defined via #[locale] + let (parent, localizer) = match ast.data { syn::Data::Struct(syn::DataStruct { fields: syn::Fields::Named(ref fields), .. - }) => fields - .named - .iter() - .find(|f| f.ident.as_ref().filter(|name| *name == "_parent").is_some()) - .map(|f| &f.ty), - _ => None, + }) => { + let named = &fields.named; + ( + named + .iter() + .find(|f| f.ident.as_ref().filter(|name| *name == "_parent").is_some()) + .map(|f| &f.ty), + { + let localizers: Vec<_> = named + .iter() + .filter(|f| f.ident.is_some()) + .flat_map(|f| { + f.attrs + .iter() + .filter(|a| a.path.is_ident("localizer")) + .map(move |_| (f.ident.to_owned().unwrap(), &f.ty)) + }) + .collect(); + if localizers.len() > 1 { + panic!("Can't have multiple localizers for a single template!"); + } else { + localizers.get(0).map(|l| l.to_owned()) + } + }, + ) + } + _ => (None, None), }; if parent.is_some() { @@ -119,6 +142,7 @@ impl TemplateInput<'_> { mime_type, parent, path, + localizer }) } From 858ea69d4aee34042b861f1073f2aaf86b9480a0 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 21:56:03 +0200 Subject: [PATCH 08/57] added locale attribute --- askama_derive/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/askama_derive/src/lib.rs b/askama_derive/src/lib.rs index 2acf58380..b8eedcb75 100644 --- a/askama_derive/src/lib.rs +++ b/askama_derive/src/lib.rs @@ -14,7 +14,7 @@ mod heritage; mod input; mod parser; -#[proc_macro_derive(Template, attributes(template))] +#[proc_macro_derive(Template, attributes(template,locale))] pub fn derive_template(input: TokenStream) -> TokenStream { generator::derive_template(input) } From 465483dec0001138f0a56805c1df14202a4a9e90 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 21:56:43 +0200 Subject: [PATCH 09/57] corrected locale attribute --- testing/tests/i18n.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index f418b6204..ba32f9c10 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -19,7 +19,7 @@ init_translation! { #[derive(Template)] #[template(path = "i18n_invalid.html")] struct UsesI18nInvalid<'a> { - #[localizer] + #[locale] loc: MyLocalizer, name: &'a str, } @@ -27,7 +27,7 @@ struct UsesI18nInvalid<'a> { #[derive(Template)] #[template(path = "i18n.html")] struct UsesI18n<'a> { - #[localizer] + #[locale] loc: MyLocalizer, name: &'a str, hours: f64, @@ -35,7 +35,7 @@ struct UsesI18n<'a> { #[derive(Template)] #[template(path = "i18n_no_args.html")] struct UsesNoArgsI18n<'a> { - #[localizer] + #[locale] loc: MyLocalizer, test: &'a str, } From 39a137031145781f32f2fee18d84e170785b1145 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 22:01:18 +0200 Subject: [PATCH 10/57] added feature localization --- askama/Cargo.toml | 1 + askama_derive/Cargo.toml | 1 + testing/Cargo.toml | 1 + 3 files changed, 3 insertions(+) diff --git a/askama/Cargo.toml b/askama/Cargo.toml index f580864a0..1f65c2250 100644 --- a/askama/Cargo.toml +++ b/askama/Cargo.toml @@ -31,6 +31,7 @@ with-mendes = ["askama_derive/with-mendes"] with-rocket = ["askama_derive/with-rocket"] with-tide = ["askama_derive/with-tide"] with-warp = ["askama_derive/with-warp"] +localization = ["dep:luent-templates","dep:unic-langid"] # deprecated mime = [] diff --git a/askama_derive/Cargo.toml b/askama_derive/Cargo.toml index ce7abfa36..ceae7e546 100644 --- a/askama_derive/Cargo.toml +++ b/askama_derive/Cargo.toml @@ -27,6 +27,7 @@ with-mendes = [] with-rocket = [] with-tide = [] with-warp = [] +localization = [] [dependencies] mime = "0.3" diff --git a/testing/Cargo.toml b/testing/Cargo.toml index 77ab25e03..f14726588 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -10,6 +10,7 @@ publish = false default = ["serde-json", "markdown"] serde-json = ["serde_json", "askama/serde-json"] markdown = ["comrak", "askama/markdown"] +localization = ["dep:luent-templates","dep:unic-langid"] [dependencies] askama = { path = "../askama", version = "0.11.0-beta.1" } From 47cbb5122803201e3cae34d5a7f10a452b2757be Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 22:27:23 +0200 Subject: [PATCH 11/57] added #[cfg(feature = "localization")] --- askama_derive/src/generator.rs | 8 +++++++- askama_derive/src/input.rs | 29 ++++++++++++++++++++++++++++- askama_derive/src/parser.rs | 7 ++++++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index a8dc060cc..b5701cf0b 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -7,7 +7,9 @@ use crate::CompileError; use proc_macro::TokenStream; use quote::{quote, ToTokens}; -use std::collections::{BTreeSet, HashMap}; +use std::collections::HashMap; +#[cfg(feature = "localization")] +use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::{cmp, hash, mem, str}; @@ -254,6 +256,7 @@ struct Generator<'a, S: std::hash::BuildHasher> { // If set to `suppress`, the whitespace characters will be removed by default unless `+` is // used. whitespace: WhitespaceHandling, + #[cfg(feature = "localization")] // Messages used with localize() localized_messages: BTreeSet, } @@ -277,6 +280,7 @@ impl<'a, S: std::hash::BuildHasher> Generator<'a, S> { buf_writable: vec![], named: 0, whitespace, + #[cfg(feature = "localization")] localized_messages: BTreeSet::new(), } } @@ -1295,12 +1299,14 @@ impl<'a, S: std::hash::BuildHasher> Generator<'a, S> { Expr::RustMacro(name, args) => self.visit_rust_macro(buf, name, args), Expr::Try(ref expr) => self.visit_try(buf, expr.as_ref())?, Expr::Tuple(ref exprs) => self.visit_tuple(buf, exprs)?, + #[cfg(feature = "localization")] Expr::Localize(message, attribute, ref args) => { self.visit_localize(buf, message, attribute, args)? } }) } + #[cfg(feature = "localization")] fn visit_localize( &mut self, buf: &mut Buffer, diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index b53c9d721..34ac5f546 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -18,6 +18,7 @@ pub(crate) struct TemplateInput<'a> { pub(crate) mime_type: String, pub(crate) parent: Option<&'a syn::Type>, pub(crate) path: PathBuf, + #[cfg(feature = "localization")] pub(crate) localizer: Option<(syn::Ident, &'a syn::Type)>, } @@ -66,6 +67,7 @@ impl TemplateInput<'_> { .iter() .find(|f| f.ident.as_ref().filter(|name| *name == "_parent").is_some()) .map(|f| &f.ty), + #[cfg(feature = "localization")] { let localizers: Vec<_> = named .iter() @@ -87,7 +89,31 @@ impl TemplateInput<'_> { } _ => (None, None), }; - + #[cfg(feature = "localization")] + let localizer = match ast.data { + syn::Data::Struct(syn::DataStruct { + fields: syn::Fields::Named(ref fields), + .. + }) => { + let named = &fields.named; + let localizers: Vec<_> = named + .iter() + .filter(|f| f.ident.is_some()) + .flat_map(|f| { + f.attrs + .iter() + .filter(|a| a.path.is_ident("localizer")) + .map(move |_| (f.ident.to_owned().unwrap(), &f.ty)) + }) + .collect(); + if localizers.len() > 1 { + panic!("Can't have multiple localizers for a single template!"); + } else { + localizers.get(0).map(|l| l.to_owned()) + } + }, + _ => (None, None), + }; if parent.is_some() { eprint!( " --> in struct {}\n = use of deprecated field '_parent'\n", @@ -142,6 +168,7 @@ impl TemplateInput<'_> { mime_type, parent, path, + #[cfg(feature = "localization")] localizer }) } diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index a19a95105..3fd0e816e 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -66,6 +66,7 @@ pub(crate) enum Expr<'a> { Call(Box>, Vec>), RustMacro(&'a str, &'a str), Try(Box>), + #[cfg(feature = "localization")] Localize(&'a str, Option<&'a str>, Vec<(&'a str, Expr<'a>)>), } @@ -669,6 +670,7 @@ macro_rules! expr_prec_layer { } } } +#[cfg(feature = "localization")] fn localize(i: &str) -> IResult<&str, Expr<'_>> { let (tail, (_, _, message, attribute, args, _)) = tuple(( tag("localize"), @@ -706,7 +708,8 @@ fn expr_handle_ws(i: &str) -> IResult<&str, Whitespace> { } fn expr_any(i: &str) -> IResult<&str, Expr<'_>> { let range_right = |i| pair(ws(alt((tag("..="), tag("..")))), opt(expr_or))(i); - alt(( + alt(( + #[cfg(feature = "localization")] map(localize, |expr| match expr { Expr::Localize(_, _, _) => expr, _ => panic!("localize failed: {:?}", expr), @@ -1279,6 +1282,7 @@ mod tests { fn test_invalid_block() { super::parse("{% extend \"blah\" %}", &Syntax::default()).unwrap(); } + #[cfg(feature = "localization")] #[test] fn test_parse_localize() { assert_eq!( @@ -1296,6 +1300,7 @@ mod tests { )] ); } + #[cfg(feature = "localization")] #[test] fn test_parse_nested_localize() { assert_eq!( From 1874729e805b76251724e0f7d29b9061d6aa2484 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 22:53:57 +0200 Subject: [PATCH 12/57] Fixed errors and added comment --- askama_derive/src/input.rs | 43 +++++++++----------------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index 34ac5f546..ae7608a1a 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -52,44 +52,21 @@ impl TemplateInput<'_> { return Err("must include 'ext' attribute when using 'source' attribute".into()) } }; - // Check to see if a `_parent` field was defined on the context // struct, and store the type for it for use in the code generator. - // Also gets the localizer if there is one defined via #[locale] - let (parent, localizer) = match ast.data { + let parent = match ast.data { syn::Data::Struct(syn::DataStruct { fields: syn::Fields::Named(ref fields), .. - }) => { - let named = &fields.named; - ( - named - .iter() - .find(|f| f.ident.as_ref().filter(|name| *name == "_parent").is_some()) - .map(|f| &f.ty), - #[cfg(feature = "localization")] - { - let localizers: Vec<_> = named - .iter() - .filter(|f| f.ident.is_some()) - .flat_map(|f| { - f.attrs - .iter() - .filter(|a| a.path.is_ident("localizer")) - .map(move |_| (f.ident.to_owned().unwrap(), &f.ty)) - }) - .collect(); - if localizers.len() > 1 { - panic!("Can't have multiple localizers for a single template!"); - } else { - localizers.get(0).map(|l| l.to_owned()) - } - }, - ) - } - _ => (None, None), + }) => fields + .named + .iter() + .find(|f| f.ident.as_ref().filter(|name| *name == "_parent").is_some()) + .map(|f| &f.ty), + _ => None, }; #[cfg(feature = "localization")] + // if enabled, it tries to get th localizer let localizer = match ast.data { syn::Data::Struct(syn::DataStruct { fields: syn::Fields::Named(ref fields), @@ -102,7 +79,7 @@ impl TemplateInput<'_> { .flat_map(|f| { f.attrs .iter() - .filter(|a| a.path.is_ident("localizer")) + .filter(|a| a.path.is_ident("locale")) .map(move |_| (f.ident.to_owned().unwrap(), &f.ty)) }) .collect(); @@ -112,7 +89,7 @@ impl TemplateInput<'_> { localizers.get(0).map(|l| l.to_owned()) } }, - _ => (None, None), + _ => None, }; if parent.is_some() { eprint!( From 98ff01916795c40864d1ddc92eb48f8b4e269716 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 22:54:10 +0200 Subject: [PATCH 13/57] fix cargo files --- askama/Cargo.toml | 4 +++- testing/Cargo.toml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/askama/Cargo.toml b/askama/Cargo.toml index 1f65c2250..055db6cbb 100644 --- a/askama/Cargo.toml +++ b/askama/Cargo.toml @@ -31,7 +31,7 @@ with-mendes = ["askama_derive/with-mendes"] with-rocket = ["askama_derive/with-rocket"] with-tide = ["askama_derive/with-tide"] with-warp = ["askama_derive/with-warp"] -localization = ["dep:luent-templates","dep:unic-langid"] +localization = ["dep:fluent-templates","dep:unic-langid"] # deprecated mime = [] @@ -47,6 +47,8 @@ percent-encoding = { version = "2.1.0", optional = true } serde = { version = "1.0", optional = true, features = ["derive"] } serde_json = { version = "1.0", optional = true } serde_yaml = { version = "0.8", optional = true } +fluent-templates = { version= "0.7.1", optional=true} +unic-langid = {version= "0.9.0", optional=true} [package.metadata.docs.rs] features = ["config", "humansize", "num-traits", "serde-json", "serde-yaml"] diff --git a/testing/Cargo.toml b/testing/Cargo.toml index f14726588..887a3563a 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -10,12 +10,14 @@ publish = false default = ["serde-json", "markdown"] serde-json = ["serde_json", "askama/serde-json"] markdown = ["comrak", "askama/markdown"] -localization = ["dep:luent-templates","dep:unic-langid"] +localization = ["dep:fluent-templates","dep:unic-langid"] [dependencies] askama = { path = "../askama", version = "0.11.0-beta.1" } comrak = { version = "0.13", default-features = false, optional = true } serde_json = { version = "1.0", optional = true } +fluent-templates = { version= "0.7.1", optional=true} +unic-langid = {version= "0.9.0", optional=true} [dev-dependencies] criterion = "0.3" From 07db0e78234b83fbea0e1a1c0042d1d581585850 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 22:57:08 +0200 Subject: [PATCH 14/57] fix test: test_parse_nested_localize --- askama_derive/src/parser.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 3fd0e816e..60e1ea207 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -1304,7 +1304,7 @@ mod tests { #[test] fn test_parse_nested_localize() { assert_eq!( - super::parse("{{ localize(a, v: localize(a, 2) ) }}", &Syntax::default()).unwrap(), + super::parse("{{ localize(a, v: localize(a, v: 32 + 7) ) }}", &Syntax::default()).unwrap(), vec![Node::Expr( Ws(None, None), Expr::Localize( @@ -1316,9 +1316,9 @@ mod tests { "a", None, vec![( - "", - Expr::StrLit("2")) - ] + "v", + Expr::BinOp("+", Expr::NumLit("32").into(), Expr::NumLit("7").into()) + )] ))] ) )] From 8bd252e96616a8bc56e4a5926cbfcd808cc463b7 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Thu, 16 Jun 2022 23:05:44 +0200 Subject: [PATCH 15/57] added #![cfg(feature = "localization")] to i18n tests --- testing/tests/i18n.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index ba32f9c10..cd685bcb1 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -3,7 +3,7 @@ #![cfg(feature = "with-i18n")] #![allow(unused)] */ - +#![cfg(feature = "localization")] use askama::init_translation; use askama::Template; From c96ada6da078c25d4367f55a1ad33e2c4b77da07 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 11:55:51 +0200 Subject: [PATCH 16/57] added askama::Local struct with impl --- askama/src/lib.rs | 109 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/askama/src/lib.rs b/askama/src/lib.rs index 3af97518d..2c5908c2f 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -218,3 +218,112 @@ mod tests { note = "file-level dependency tracking is handled automatically without build script" )] pub fn rerun_if_templates_changed() {} + +#[cfg(feature = "localization")] +use fluent_templates::Loader; +#[cfg(feature = "localization")] +pub struct Locale<'a>{ + loader: &'a fluent_templates::StaticLoader, + language: unic_langid::LanguageIdentifier, +} +#[cfg(feature = "localization")] +impl<'a> Locale<'a> { + pub fn new(language: unic_langid::LanguageIdentifier, templates: &'static fluent_templates::StaticLoader) -> Locale<'a> { + + Self { + loader: templates, + language, + } + } + pub fn translate( + &self, + text_id: &str, + args: + &std::collections::HashMap>, + ) -> String { + self.loader.lookup_with_args(&self.language, text_id, args) + } + +} +#[macro_export] +macro_rules! init_translation { + ( + $v: vis $n: ident { + static_loader_name: $static_loader_name: ident, + locales: $locales: expr, + fallback_language: $fallback_language: expr, + customise: $customise: expr + } + ) => { + use fluent_templates::Loader; + fluent_templates::static_loader! { + // Declare our `StaticLoader` named `LOCALES`. + static $static_loader_name = { + // The directory of localisations and fluent resources. + locales: $locales, + // The language to falback on if something is not present. + fallback_language: $fallback_language, + // Optional: A fluent resource that is shared with every locale. + //core_locales: "/core.ftl", + // Removes unicode isolating marks around arguments, you typically + // should only set to false when testing. + customise: $customise, + }; + } + $v struct $n { + language: unic_langid::LanguageIdentifier, + loader: &'static fluent_templates::once_cell::sync::Lazy + } + impl $n { + pub fn new(language: unic_langid::LanguageIdentifier) -> $n { + $n { + language, + loader: & $static_loader_name + } + } + pub fn defsault() -> $n { + $n { + language: unic_langid::langid!($fallback_language), + loader: & $static_loader_name + } + } + } + impl $n { + fn get_fallback_language(&self) -> unic_langid::LanguageIdentifier { + unic_langid::langid!($fallback_language) + } + + fn get_language(&self) -> unic_langid::LanguageIdentifier { + self.language.clone() + } + + fn translate( + &self, + text_id: &str, + args: + &std::collections::HashMap>, + ) -> String { + self.loader.lookup_with_args(&self.language, text_id, args) + } + + fn has_default_translation(&self, m: &str) -> bool { + // lookup_single_language panic's when invalid args are given + std::panic::set_hook(Box::new(|_info| { + // do nothing + })); + + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + self.loader.lookup_single_language::(&self.get_fallback_language(), m, None) + })); + + let _ = std::panic::take_hook(); + + match result { + Ok(None) => false, + _ => true + } + } + } + + } +} From d1a46fe13caf2e542012b5b86e68674132a2f43f Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 11:56:22 +0200 Subject: [PATCH 17/57] changed tests to use askama::Local --- testing/tests/i18n.rs | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index cd685bcb1..97d5bc7de 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -1,26 +1,21 @@ -// TODO -/* -#![cfg(feature = "with-i18n")] -#![allow(unused)] -*/ #![cfg(feature = "localization")] -use askama::init_translation; use askama::Template; - -init_translation! { - pub MyLocalizer { - static_loader_name: LOCALES, +use askama::Locale; +use fluent_templates::static_loader; +static_loader! { + static LOCALES = { locales: "i18n-basic", fallback_language: "en-US", - customise: |bundle| bundle.set_use_isolating(false) - } + // Removes unicode isolating marks around arguments, you typically + // should only set to false when testing. + customise: |bundle| bundle.set_use_isolating(false), + }; } - #[derive(Template)] #[template(path = "i18n_invalid.html")] struct UsesI18nInvalid<'a> { #[locale] - loc: MyLocalizer, + loc: Locale<'a>, name: &'a str, } @@ -28,7 +23,7 @@ struct UsesI18nInvalid<'a> { #[template(path = "i18n.html")] struct UsesI18n<'a> { #[locale] - loc: MyLocalizer, + loc: Locale<'a>, name: &'a str, hours: f64, } @@ -36,14 +31,14 @@ struct UsesI18n<'a> { #[template(path = "i18n_no_args.html")] struct UsesNoArgsI18n<'a> { #[locale] - loc: MyLocalizer, + loc: Locale<'a>, test: &'a str, } #[test] fn existing_language() { let template = UsesI18n { - loc: MyLocalizer::new(unic_langid::langid!("es-MX")), + loc: Locale::new(unic_langid::langid!("es-MX"), &LOCALES), name: "Hilda", hours: 300072.3, }; @@ -57,7 +52,7 @@ fn existing_language() { #[test] fn unknown_language() { let template = UsesI18n { - loc: MyLocalizer::new(unic_langid::langid!("nl-BE")), + loc: Locale::new(unic_langid::langid!("nl-BE"), &LOCALES), name: "Hilda", hours: 300072.3, }; @@ -71,7 +66,7 @@ fn unknown_language() { #[test] fn no_args() { let template = UsesNoArgsI18n { - loc: MyLocalizer::new(unic_langid::langid!("es-MX")), + loc: Locale::new(unic_langid::langid!("es-MX"), &LOCALES), test: "" }; assert_eq!( From cdbc7d0cf832c71364d9f7907a59192f5f45e90b Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 11:56:53 +0200 Subject: [PATCH 18/57] removed init_translation macro since its not needed anymore --- askama/src/lib.rs | 84 +---------------------------------------------- 1 file changed, 1 insertion(+), 83 deletions(-) diff --git a/askama/src/lib.rs b/askama/src/lib.rs index 2c5908c2f..b745dcff5 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -244,86 +244,4 @@ impl<'a> Locale<'a> { self.loader.lookup_with_args(&self.language, text_id, args) } -} -#[macro_export] -macro_rules! init_translation { - ( - $v: vis $n: ident { - static_loader_name: $static_loader_name: ident, - locales: $locales: expr, - fallback_language: $fallback_language: expr, - customise: $customise: expr - } - ) => { - use fluent_templates::Loader; - fluent_templates::static_loader! { - // Declare our `StaticLoader` named `LOCALES`. - static $static_loader_name = { - // The directory of localisations and fluent resources. - locales: $locales, - // The language to falback on if something is not present. - fallback_language: $fallback_language, - // Optional: A fluent resource that is shared with every locale. - //core_locales: "/core.ftl", - // Removes unicode isolating marks around arguments, you typically - // should only set to false when testing. - customise: $customise, - }; - } - $v struct $n { - language: unic_langid::LanguageIdentifier, - loader: &'static fluent_templates::once_cell::sync::Lazy - } - impl $n { - pub fn new(language: unic_langid::LanguageIdentifier) -> $n { - $n { - language, - loader: & $static_loader_name - } - } - pub fn defsault() -> $n { - $n { - language: unic_langid::langid!($fallback_language), - loader: & $static_loader_name - } - } - } - impl $n { - fn get_fallback_language(&self) -> unic_langid::LanguageIdentifier { - unic_langid::langid!($fallback_language) - } - - fn get_language(&self) -> unic_langid::LanguageIdentifier { - self.language.clone() - } - - fn translate( - &self, - text_id: &str, - args: - &std::collections::HashMap>, - ) -> String { - self.loader.lookup_with_args(&self.language, text_id, args) - } - - fn has_default_translation(&self, m: &str) -> bool { - // lookup_single_language panic's when invalid args are given - std::panic::set_hook(Box::new(|_info| { - // do nothing - })); - - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - self.loader.lookup_single_language::(&self.get_fallback_language(), m, None) - })); - - let _ = std::panic::take_hook(); - - match result { - Ok(None) => false, - _ => true - } - } - } - - } -} +} \ No newline at end of file From 4c7300c4c1637e3a9bc8b903117f6045bb21b293 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 11:58:19 +0200 Subject: [PATCH 19/57] refactored no args test adn its template --- testing/templates/i18n_no_args.html | 2 +- testing/tests/i18n.rs | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/testing/templates/i18n_no_args.html b/testing/templates/i18n_no_args.html index f72083b06..30075c599 100644 --- a/testing/templates/i18n_no_args.html +++ b/testing/templates/i18n_no_args.html @@ -1 +1 @@ -

{{ localize(test, test: test) }}

\ No newline at end of file +

{{ localize(test, test: "") }}

\ No newline at end of file diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index 97d5bc7de..34d7bb702 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -32,7 +32,6 @@ struct UsesI18n<'a> { struct UsesNoArgsI18n<'a> { #[locale] loc: Locale<'a>, - test: &'a str, } #[test] @@ -66,8 +65,7 @@ fn unknown_language() { #[test] fn no_args() { let template = UsesNoArgsI18n { - loc: Locale::new(unic_langid::langid!("es-MX"), &LOCALES), - test: "" + loc: Locale::new(unic_langid::langid!("en-US"), &LOCALES), }; assert_eq!( template.render().unwrap(), From 17f95e740c0fdc62100a0481c2354157f640ca5d Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 12:15:33 +0200 Subject: [PATCH 20/57] removed last todos --- askama_derive/src/generator.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index 0cc969a58..7131b3016 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -1317,15 +1317,6 @@ impl<'a> Generator<'a> { attribute: Option<&str>, args: &[(&str, Expr<'_>)], ) -> Result { - /* TODO - if !cfg!(feature = "with-i18n") { - panic!( - "The askama feature 'with-i18n' must be activated to enable calling `localize`." - ); - } - */ - - // TODO let localizer = self.input.localizer.as_ref().expect( "A template struct must have a member with the `#[locale]` \ attribute that implements `askama::Localize` to enable calling the localize() filter", From 2c827f2f33a31ca3d611ba8cea27d98921d5a44c Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 12:37:20 +0200 Subject: [PATCH 21/57] Added test for invalid tags with no fallack language --- testing/templates/i18n_broken.html | 1 + testing/tests/i18n.rs | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 testing/templates/i18n_broken.html diff --git a/testing/templates/i18n_broken.html b/testing/templates/i18n_broken.html new file mode 100644 index 000000000..0ea2b74fd --- /dev/null +++ b/testing/templates/i18n_broken.html @@ -0,0 +1 @@ +

{{ localize(car, color: car_color) }}

\ No newline at end of file diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index 34d7bb702..8d386574a 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -33,6 +33,13 @@ struct UsesNoArgsI18n<'a> { #[locale] loc: Locale<'a>, } +#[derive(Template)] +#[template(path = "i18n_broken.html")] +struct InvalidI18n<'a> { + #[locale] + loc: Locale<'a>, + car_color: &'a str +} #[test] fn existing_language() { @@ -49,7 +56,7 @@ fn existing_language() { } #[test] -fn unknown_language() { +fn fallback_language() { let template = UsesI18n { loc: Locale::new(unic_langid::langid!("nl-BE"), &LOCALES), name: "Hilda", @@ -72,3 +79,12 @@ fn no_args() { r#"

This is a test

"# ) } + +#[test] +fn invalid_tags_language() { + let template = InvalidI18n { + loc: Locale::new(unic_langid::langid!("nl-BE"), &LOCALES), + car_color: "Red" + }; + assert_eq!(template.render().unwrap(), r#"

Unknown localization car

"#); // Should panic here +} \ No newline at end of file From c3f5684f7ba77f5b9eef1499b96eaa65783cf317 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 12:55:07 +0200 Subject: [PATCH 22/57] fixed lint error from Lint workflow --- askama/src/lib.rs | 20 ++++++++++---------- askama_derive/src/generator.rs | 3 +-- askama_derive/src/input.rs | 4 ++-- askama_derive/src/lib.rs | 2 +- askama_derive/src/parser.rs | 27 ++++++++++++++++++--------- testing/tests/i18n.rs | 19 +++++++++---------- 6 files changed, 41 insertions(+), 34 deletions(-) diff --git a/askama/src/lib.rs b/askama/src/lib.rs index b745dcff5..e0915642a 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -222,26 +222,26 @@ pub fn rerun_if_templates_changed() {} #[cfg(feature = "localization")] use fluent_templates::Loader; #[cfg(feature = "localization")] -pub struct Locale<'a>{ +pub struct Locale<'a> { loader: &'a fluent_templates::StaticLoader, language: unic_langid::LanguageIdentifier, } #[cfg(feature = "localization")] impl<'a> Locale<'a> { - pub fn new(language: unic_langid::LanguageIdentifier, templates: &'static fluent_templates::StaticLoader) -> Locale<'a> { - - Self { - loader: templates, - language, + pub fn new( + language: unic_langid::LanguageIdentifier, + templates: &'static fluent_templates::StaticLoader, + ) -> Locale<'a> { + Self { + loader: templates, + language, } } pub fn translate( &self, text_id: &str, - args: - &std::collections::HashMap>, + args: &std::collections::HashMap>, ) -> String { self.loader.lookup_with_args(&self.language, text_id, args) } - -} \ No newline at end of file +} diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index 7131b3016..41ab0f10c 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -7,9 +7,9 @@ use crate::CompileError; use proc_macro::TokenStream; use quote::{quote, ToTokens}; -use std::collections::HashMap; #[cfg(feature = "localization")] use std::collections::BTreeSet; +use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::{cmp, hash, mem, str}; @@ -1355,7 +1355,6 @@ impl<'a> Generator<'a> { Ok(DisplayWrap::Unwrapped) } - fn visit_try( &mut self, buf: &mut Buffer, diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index ae7608a1a..5a99c7ee4 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -88,7 +88,7 @@ impl TemplateInput<'_> { } else { localizers.get(0).map(|l| l.to_owned()) } - }, + } _ => None, }; if parent.is_some() { @@ -146,7 +146,7 @@ impl TemplateInput<'_> { parent, path, #[cfg(feature = "localization")] - localizer + localizer, }) } diff --git a/askama_derive/src/lib.rs b/askama_derive/src/lib.rs index b8eedcb75..553da5037 100644 --- a/askama_derive/src/lib.rs +++ b/askama_derive/src/lib.rs @@ -14,7 +14,7 @@ mod heritage; mod input; mod parser; -#[proc_macro_derive(Template, attributes(template,locale))] +#[proc_macro_derive(Template, attributes(template, locale))] pub fn derive_template(input: TokenStream) -> TokenStream { generator::derive_template(input) } diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 60e1ea207..fb192014d 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -708,7 +708,7 @@ fn expr_handle_ws(i: &str) -> IResult<&str, Whitespace> { } fn expr_any(i: &str) -> IResult<&str, Expr<'_>> { let range_right = |i| pair(ws(alt((tag("..="), tag("..")))), opt(expr_or))(i); - alt(( + alt(( #[cfg(feature = "localization")] map(localize, |expr| match expr { Expr::Localize(_, _, _) => expr, @@ -1304,7 +1304,11 @@ mod tests { #[test] fn test_parse_nested_localize() { assert_eq!( - super::parse("{{ localize(a, v: localize(a, v: 32 + 7) ) }}", &Syntax::default()).unwrap(), + super::parse( + "{{ localize(a, v: localize(a, v: 32 + 7) ) }}", + &Syntax::default() + ) + .unwrap(), vec![Node::Expr( Ws(None, None), Expr::Localize( @@ -1313,13 +1317,18 @@ mod tests { vec![( "v", Expr::Localize( - "a", - None, - vec![( - "v", - Expr::BinOp("+", Expr::NumLit("32").into(), Expr::NumLit("7").into()) - )] - ))] + "a", + None, + vec![( + "v", + Expr::BinOp( + "+", + Expr::NumLit("32").into(), + Expr::NumLit("7").into() + ) + )] + ) + )] ) )] ); diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index 8d386574a..13e59c66d 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -1,6 +1,6 @@ #![cfg(feature = "localization")] -use askama::Template; use askama::Locale; +use askama::Template; use fluent_templates::static_loader; static_loader! { static LOCALES = { @@ -36,9 +36,8 @@ struct UsesNoArgsI18n<'a> { #[derive(Template)] #[template(path = "i18n_broken.html")] struct InvalidI18n<'a> { - #[locale] loc: Locale<'a>, - car_color: &'a str + car_color: &'a str, } #[test] @@ -74,17 +73,17 @@ fn no_args() { let template = UsesNoArgsI18n { loc: Locale::new(unic_langid::langid!("en-US"), &LOCALES), }; - assert_eq!( - template.render().unwrap(), - r#"

This is a test

"# - ) + assert_eq!(template.render().unwrap(), r#"

This is a test

"#) } #[test] fn invalid_tags_language() { let template = InvalidI18n { loc: Locale::new(unic_langid::langid!("nl-BE"), &LOCALES), - car_color: "Red" + car_color: "Red", }; - assert_eq!(template.render().unwrap(), r#"

Unknown localization car

"#); // Should panic here -} \ No newline at end of file + assert_eq!( + template.render().unwrap(), + r#"

Unknown localization car

"# + ); +} From 2658da37d104e18dc47279084b85dc4eee0f7a65 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 13:21:49 +0200 Subject: [PATCH 23/57] Fixed typo th -> the --- askama_derive/src/input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index 5a99c7ee4..dfc6716f7 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -66,7 +66,7 @@ impl TemplateInput<'_> { _ => None, }; #[cfg(feature = "localization")] - // if enabled, it tries to get th localizer + // if enabled, it tries to get the localizer let localizer = match ast.data { syn::Data::Struct(syn::DataStruct { fields: syn::Fields::Named(ref fields), From 7b89872b045876994a3314edb0fd10e26becfdc1 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 13:41:40 +0200 Subject: [PATCH 24/57] fixed test I messed up --- testing/tests/i18n.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index 13e59c66d..d1577fa68 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -36,6 +36,7 @@ struct UsesNoArgsI18n<'a> { #[derive(Template)] #[template(path = "i18n_broken.html")] struct InvalidI18n<'a> { + #[locale] loc: Locale<'a>, car_color: &'a str, } From 5ce38a3ec2d97d9949131606a758eb76a634df4c Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 14:53:27 +0200 Subject: [PATCH 25/57] Added fn quoted_ident to support only localize("foo", bar:baz) and added fail() if localize() is detected in a template while the localization feature is not activated --- askama_derive/src/parser.rs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index fb192014d..96c0c6ede 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -2,11 +2,11 @@ use std::cell::Cell; use std::str; use nom::branch::alt; -use nom::bytes::complete::{escaped, is_not, tag, take_till, take_until}; +use nom::bytes::complete::{escaped, is_not, tag, take_till, take_until, take}; use nom::character::complete::{anychar, char, digit1}; -use nom::combinator::{complete, consumed, cut, eof, map, not, opt, peek, recognize, value}; +use nom::combinator::{complete, consumed, cut, eof, map, not, opt, peek, recognize, value, fail}; use nom::error::{Error, ErrorKind}; -use nom::multi::{fold_many0, many0, many1, separated_list0, separated_list1}; +use nom::multi::{fold_many0, many0, many1, separated_list0, separated_list1, many_till}; use nom::sequence::{delimited, pair, preceded, terminated, tuple}; use nom::{self, error_position, AsChar, IResult, InputTakeAtPosition}; @@ -671,11 +671,22 @@ macro_rules! expr_prec_layer { } } #[cfg(feature = "localization")] +fn quoted_ident(input: &str) -> IResult<&str, &str> { + // Match " + let (remaining, _) = tag("\"")(input)?; + let (remaining, (inner, _)) = many_till(take(1u32), tag("\""))(remaining)?; + + // Extract inner range + let length = inner.len(); + println!("{}", &input[1 .. length+1]); + Ok((remaining, &input[1 .. length+1])) +} +#[cfg(feature = "localization")] fn localize(i: &str) -> IResult<&str, Expr<'_>> { - let (tail, (_, _, message, attribute, args, _)) = tuple(( + let (tail, (_, _,message, attribute, args, _)) = tuple(( tag("localize"), ws(tag("(")), - identifier, + quoted_ident, opt(tuple((ws(tag(".")), identifier))), opt(tuple(( ws(tag(",")), @@ -714,6 +725,8 @@ fn expr_any(i: &str) -> IResult<&str, Expr<'_>> { Expr::Localize(_, _, _) => expr, _ => panic!("localize failed: {:?}", expr), }), + #[cfg(not(feature = "localization"))] + fail, map(range_right, |(op, right)| { Expr::Range(op, None, right.map(Box::new)) }), @@ -1286,7 +1299,7 @@ mod tests { #[test] fn test_parse_localize() { assert_eq!( - super::parse("{{ localize(a, v: 32 + 7) }}", &Syntax::default()).unwrap(), + super::parse(r#"{{ localize("a", v: 32 + 7) }}"#, &Syntax::default()).unwrap(), vec![Node::Expr( Ws(None, None), Expr::Localize( @@ -1305,7 +1318,7 @@ mod tests { fn test_parse_nested_localize() { assert_eq!( super::parse( - "{{ localize(a, v: localize(a, v: 32 + 7) ) }}", + r#"{{ localize("a", v: localize("a", v: 32 + 7) ) }}"#, &Syntax::default() ) .unwrap(), From 7a11db61a1cbe9eb47593cc7680a24595ef45a8c Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 14:54:12 +0200 Subject: [PATCH 26/57] Updated test templates -> quoted all messages --- testing/templates/i18n.html | 4 ++-- testing/templates/i18n_broken.html | 2 +- testing/templates/i18n_invalid.html | 4 ++-- testing/templates/i18n_no_args.html | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/testing/templates/i18n.html b/testing/templates/i18n.html index d73e64660..76f116ad6 100644 --- a/testing/templates/i18n.html +++ b/testing/templates/i18n.html @@ -1,2 +1,2 @@ -

{{ localize(greeting, name: name) }}

-

{{ localize(age, hours: hours ) }}

\ No newline at end of file +

{{ localize("greeting", name: name) }}

+

{{ localize("age", hours: hours ) }}

\ No newline at end of file diff --git a/testing/templates/i18n_broken.html b/testing/templates/i18n_broken.html index 0ea2b74fd..983f484e1 100644 --- a/testing/templates/i18n_broken.html +++ b/testing/templates/i18n_broken.html @@ -1 +1 @@ -

{{ localize(car, color: car_color) }}

\ No newline at end of file +

{{ localize("car", color: car_color) }}

\ No newline at end of file diff --git a/testing/templates/i18n_invalid.html b/testing/templates/i18n_invalid.html index 5e1262048..f5afefa58 100644 --- a/testing/templates/i18n_invalid.html +++ b/testing/templates/i18n_invalid.html @@ -1,2 +1,2 @@ -

{{ localize(greetingsss, name: name) }}

-

{{ localize(ages) }}

\ No newline at end of file +

{{ localize("greetingsss", name: name) }}

+

{{ localize("ages") }}

\ No newline at end of file diff --git a/testing/templates/i18n_no_args.html b/testing/templates/i18n_no_args.html index 30075c599..a16fadd08 100644 --- a/testing/templates/i18n_no_args.html +++ b/testing/templates/i18n_no_args.html @@ -1 +1 @@ -

{{ localize(test, test: "") }}

\ No newline at end of file +

{{ localize("test", test: "") }}

\ No newline at end of file From 355163a594b347c5843141d13dc6ddaf8b26f974 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 14:55:06 +0200 Subject: [PATCH 27/57] Added cut() to localze function --- askama_derive/src/parser.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 96c0c6ede..368ef66e6 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -683,7 +683,7 @@ fn quoted_ident(input: &str) -> IResult<&str, &str> { } #[cfg(feature = "localization")] fn localize(i: &str) -> IResult<&str, Expr<'_>> { - let (tail, (_, _,message, attribute, args, _)) = tuple(( + let (tail, (_, _,message, attribute, args, _)) = cut(tuple(( tag("localize"), ws(tag("(")), quoted_ident, @@ -693,7 +693,7 @@ fn localize(i: &str) -> IResult<&str, Expr<'_>> { separated_list0(ws(tag(",")), tuple((identifier, ws(tag(":")), expr_any))), ))), ws(tag(")")), - ))(i)?; + )))(i)?; Ok(( tail, Expr::Localize( From 62027c8d6ab2c948540fda16f10072decd120d46 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 15:13:04 +0200 Subject: [PATCH 28/57] Revert "Added cut() to localze function" because expr_any uses alt which tries until on parser succeeds but this also means when localize gets call with non localize string it fails and generates a failure This reverts commit 355163a594b347c5843141d13dc6ddaf8b26f974. --- askama_derive/src/parser.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 368ef66e6..96c0c6ede 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -683,7 +683,7 @@ fn quoted_ident(input: &str) -> IResult<&str, &str> { } #[cfg(feature = "localization")] fn localize(i: &str) -> IResult<&str, Expr<'_>> { - let (tail, (_, _,message, attribute, args, _)) = cut(tuple(( + let (tail, (_, _,message, attribute, args, _)) = tuple(( tag("localize"), ws(tag("(")), quoted_ident, @@ -693,7 +693,7 @@ fn localize(i: &str) -> IResult<&str, Expr<'_>> { separated_list0(ws(tag(",")), tuple((identifier, ws(tag(":")), expr_any))), ))), ws(tag(")")), - )))(i)?; + ))(i)?; Ok(( tail, Expr::Localize( From bbe59913005537167c00aab820b1eb251ba29de9 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 15:31:27 +0200 Subject: [PATCH 29/57] HashMap to HashMap<&str, ...> --- askama/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/askama/src/lib.rs b/askama/src/lib.rs index e0915642a..e5ade9b7f 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -240,7 +240,7 @@ impl<'a> Locale<'a> { pub fn translate( &self, text_id: &str, - args: &std::collections::HashMap>, + args: &std::collections::HashMap<&str, fluent_templates::fluent_bundle::FluentValue<'_>>, ) -> String { self.loader.lookup_with_args(&self.language, text_id, args) } From f3ec8c7b86f7a6de647893c0a5fe91e2f2cdfe8c Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 15:31:42 +0200 Subject: [PATCH 30/57] removed .to_string() --- askama_derive/src/generator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index 41ab0f10c..a1b24b476 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -1345,7 +1345,7 @@ impl<'a> Generator<'a> { buf.write(", "); } buf.write(&format!( - "(\"{}\".to_string(), ({}).into())", + "(\"{}\", ({}).into())", name, self.visit_expr_root(value)? )); From cc01d9be6eeab25d6431ba744e898c2923ddfeee Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Fri, 17 Jun 2022 15:33:44 +0200 Subject: [PATCH 31/57] removed "dep:" from toml files and removed println --- askama/Cargo.toml | 2 +- askama_derive/src/parser.rs | 1 - testing/Cargo.toml | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/askama/Cargo.toml b/askama/Cargo.toml index 055db6cbb..ce3885cd4 100644 --- a/askama/Cargo.toml +++ b/askama/Cargo.toml @@ -31,7 +31,7 @@ with-mendes = ["askama_derive/with-mendes"] with-rocket = ["askama_derive/with-rocket"] with-tide = ["askama_derive/with-tide"] with-warp = ["askama_derive/with-warp"] -localization = ["dep:fluent-templates","dep:unic-langid"] +localization = ["fluent-templates","unic-langid"] # deprecated mime = [] diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 96c0c6ede..8d0459db3 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -678,7 +678,6 @@ fn quoted_ident(input: &str) -> IResult<&str, &str> { // Extract inner range let length = inner.len(); - println!("{}", &input[1 .. length+1]); Ok((remaining, &input[1 .. length+1])) } #[cfg(feature = "localization")] diff --git a/testing/Cargo.toml b/testing/Cargo.toml index 887a3563a..84ad48bf2 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -10,7 +10,7 @@ publish = false default = ["serde-json", "markdown"] serde-json = ["serde_json", "askama/serde-json"] markdown = ["comrak", "askama/markdown"] -localization = ["dep:fluent-templates","dep:unic-langid"] +localization = ["fluent-templates","unic-langid"] [dependencies] askama = { path = "../askama", version = "0.11.0-beta.1" } From afbc64f6817c070f681d47e93e6743aa89c7bf56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Kijewski?= Date: Fri, 17 Jun 2022 22:06:23 +0200 Subject: [PATCH 32/57] A bunch of changes * Format the code, and add missing newlines between blocks * Collect arguments in Vec instead of HashMap to preserve call order * Let text_id be a normal variable or literal * Fix cut() in parser * Don't feature gate most on the implementation to make the implementation simpler to read and understand * Add more error messages how to use the feature * Build the hashmap in one place --- askama/Cargo.toml | 6 +- askama/src/lib.rs | 37 ++++---- askama_derive/Cargo.toml | 6 +- askama_derive/src/generator.rs | 61 ++++--------- askama_derive/src/input.rs | 43 +++++---- askama_derive/src/parser.rs | 160 ++++++++++++++++++--------------- testing/Cargo.toml | 8 +- testing/tests/i18n.rs | 5 ++ 8 files changed, 165 insertions(+), 161 deletions(-) diff --git a/askama/Cargo.toml b/askama/Cargo.toml index ce3885cd4..f121c3030 100644 --- a/askama/Cargo.toml +++ b/askama/Cargo.toml @@ -19,6 +19,7 @@ maintenance = { status = "actively-developed" } default = ["config", "humansize", "num-traits", "urlencode"] config = ["askama_derive/config"] humansize = ["askama_derive/humansize", "dep_humansize"] +localization = ["askama_derive/localization", "fluent-templates", "unic-langid"] markdown = ["askama_derive/markdown", "comrak"] num-traits = ["askama_derive/num-traits", "dep_num_traits"] serde-json = ["askama_derive/serde-json", "askama_escape/json", "serde", "serde_json"] @@ -31,7 +32,6 @@ with-mendes = ["askama_derive/with-mendes"] with-rocket = ["askama_derive/with-rocket"] with-tide = ["askama_derive/with-tide"] with-warp = ["askama_derive/with-warp"] -localization = ["fluent-templates","unic-langid"] # deprecated mime = [] @@ -43,12 +43,12 @@ askama_escape = { version = "0.10.3", path = "../askama_escape" } comrak = { version = "0.13", optional = true, default-features = false } dep_humansize = { package = "humansize", version = "1.1.0", optional = true } dep_num_traits = { package = "num-traits", version = "0.2.6", optional = true } +fluent-templates = { version = "0.7.1", optional = true, default-features = false } percent-encoding = { version = "2.1.0", optional = true } serde = { version = "1.0", optional = true, features = ["derive"] } serde_json = { version = "1.0", optional = true } serde_yaml = { version = "0.8", optional = true } -fluent-templates = { version= "0.7.1", optional=true} -unic-langid = {version= "0.9.0", optional=true} +unic-langid = { version= "0.9.0", optional = true} [package.metadata.docs.rs] features = ["config", "humansize", "num-traits", "serde-json", "serde-yaml"] diff --git a/askama/src/lib.rs b/askama/src/lib.rs index e5ade9b7f..0fbdd69f0 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -71,6 +71,11 @@ use std::fmt; pub use askama_derive::Template; pub use askama_escape::{Html, MarkupDisplay, Text}; +#[cfg(feature = "localization")] +#[doc(hidden)] +pub use fluent_templates::fluent_bundle::FluentValue; +#[cfg(feature = "localization")] +use fluent_templates::{Loader, StaticLoader}; #[doc(hidden)] pub use crate as shared; @@ -219,29 +224,31 @@ mod tests { )] pub fn rerun_if_templates_changed() {} -#[cfg(feature = "localization")] -use fluent_templates::Loader; #[cfg(feature = "localization")] pub struct Locale<'a> { - loader: &'a fluent_templates::StaticLoader, + loader: &'a StaticLoader, language: unic_langid::LanguageIdentifier, } + #[cfg(feature = "localization")] -impl<'a> Locale<'a> { - pub fn new( - language: unic_langid::LanguageIdentifier, - templates: &'static fluent_templates::StaticLoader, - ) -> Locale<'a> { - Self { - loader: templates, - language, - } +impl Locale<'_> { + pub fn new(language: unic_langid::LanguageIdentifier, loader: &'static StaticLoader) -> Self { + Self { loader, language } } - pub fn translate( + + pub fn translate<'a>( &self, text_id: &str, - args: &std::collections::HashMap<&str, fluent_templates::fluent_bundle::FluentValue<'_>>, + args: impl IntoIterator)>, ) -> String { - self.loader.lookup_with_args(&self.language, text_id, args) + use std::collections::HashMap; + use std::iter::FromIterator; + + let args = HashMap::<&str, FluentValue<'_>>::from_iter(args); + let args = match args.is_empty() { + true => None, + false => Some(&args), + }; + self.loader.lookup_complete(&self.language, text_id, args) } } diff --git a/askama_derive/Cargo.toml b/askama_derive/Cargo.toml index ceae7e546..0c7931f7d 100644 --- a/askama_derive/Cargo.toml +++ b/askama_derive/Cargo.toml @@ -15,11 +15,12 @@ proc-macro = true [features] config = ["serde", "toml"] humansize = [] +localization = [] markdown = [] -urlencode = [] +num-traits = [] serde-json = [] serde-yaml = [] -num-traits = [] +urlencode = [] with-actix-web = [] with-axum = [] with-gotham = [] @@ -27,7 +28,6 @@ with-mendes = [] with-rocket = [] with-tide = [] with-warp = [] -localization = [] [dependencies] mime = "0.3" diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index a1b24b476..2fa2805c8 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -7,8 +7,6 @@ use crate::CompileError; use proc_macro::TokenStream; use quote::{quote, ToTokens}; -#[cfg(feature = "localization")] -use std::collections::BTreeSet; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::{cmp, hash, mem, str}; @@ -257,9 +255,6 @@ struct Generator<'a> { // If set to `suppress`, the whitespace characters will be removed by default unless `+` is // used. whitespace: WhitespaceHandling, - #[cfg(feature = "localization")] - // Messages used with localize() - localized_messages: BTreeSet, } impl<'a> Generator<'a> { @@ -281,8 +276,6 @@ impl<'a> Generator<'a> { buf_writable: vec![], named: 0, whitespace, - #[cfg(feature = "localization")] - localized_messages: BTreeSet::new(), } } @@ -1302,55 +1295,35 @@ impl<'a> Generator<'a> { Expr::RustMacro(name, args) => self.visit_rust_macro(buf, name, args), Expr::Try(ref expr) => self.visit_try(buf, expr.as_ref())?, Expr::Tuple(ref exprs) => self.visit_tuple(buf, exprs)?, - #[cfg(feature = "localization")] - Expr::Localize(message, attribute, ref args) => { - self.visit_localize(buf, message, attribute, args)? - } + Expr::Localize(ref message, ref args) => self.visit_localize(buf, message, args)?, }) } - #[cfg(feature = "localization")] fn visit_localize( &mut self, buf: &mut Buffer, - message: &str, - attribute: Option<&str>, + message: &Expr<'_>, args: &[(&str, Expr<'_>)], ) -> Result { - let localizer = self.input.localizer.as_ref().expect( - "A template struct must have a member with the `#[locale]` \ - attribute that implements `askama::Localize` to enable calling the localize() filter", - ); - - let mut message = message.to_string(); - if let Some(attribute) = attribute { - message.push_str("."); - message.push_str(attribute); - } - - assert!( - message.chars().find(|c| *c == '"').is_none(), - "message ids with quotes in them break the generator, please remove" - ); - - self.localized_messages.insert(message.clone()); + let localizer = + self.input.localizer.as_deref().ok_or( + "You have to annotate a field with #[locale] to use the localize() function.", + )?; buf.write(&format!( - "self.{}.translate(\"{}\", &std::iter::FromIterator::from_iter(vec![", - localizer.0, message + "self.{}.translate(", + normalize_identifier(localizer) )); - - for (i, (name, value)) in args.iter().enumerate() { - if i > 0 { - buf.write(", "); - } - buf.write(&format!( - "(\"{}\", ({}).into())", - name, - self.visit_expr_root(value)? - )); + self.visit_expr(buf, message)?; + buf.writeln(", [")?; + buf.indent(); + for (k, v) in args { + buf.write(&format!("({:?}, ::askama::FluentValue::from(", k)); + self.visit_expr(buf, v)?; + buf.writeln(")),")?; } - buf.write("]))"); + buf.dedent()?; + buf.write("])"); Ok(DisplayWrap::Unwrapped) } diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index dfc6716f7..892b2b0b7 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -18,8 +18,7 @@ pub(crate) struct TemplateInput<'a> { pub(crate) mime_type: String, pub(crate) parent: Option<&'a syn::Type>, pub(crate) path: PathBuf, - #[cfg(feature = "localization")] - pub(crate) localizer: Option<(syn::Ident, &'a syn::Type)>, + pub(crate) localizer: Option, } impl TemplateInput<'_> { @@ -65,32 +64,38 @@ impl TemplateInput<'_> { .map(|f| &f.ty), _ => None, }; - #[cfg(feature = "localization")] - // if enabled, it tries to get the localizer + let localizer = match ast.data { syn::Data::Struct(syn::DataStruct { fields: syn::Fields::Named(ref fields), .. }) => { - let named = &fields.named; - let localizers: Vec<_> = named - .iter() - .filter(|f| f.ident.is_some()) - .flat_map(|f| { - f.attrs - .iter() - .filter(|a| a.path.is_ident("locale")) - .map(move |_| (f.ident.to_owned().unwrap(), &f.ty)) - }) - .collect(); - if localizers.len() > 1 { - panic!("Can't have multiple localizers for a single template!"); - } else { - localizers.get(0).map(|l| l.to_owned()) + let mut localizers = + fields + .named + .iter() + .filter(|&f| f.ident.is_some()) + .flat_map( + |f| match f.attrs.iter().any(|a| a.path.is_ident("locale")) { + true => Some(f.ident.as_ref()?.to_string()), + false => None, + }, + ); + match localizers.next() { + Some(localizer) => { + if !cfg!(feature = "localization") { + return Err("You have to active the \"localization\" feature to use #[locale] on fields.".into()); + } else if localizers.next().is_some() { + return Err("You cannot mark more than one field as #[locale].".into()); + } + Some(localizer) + } + None => None, } } _ => None, }; + if parent.is_some() { eprint!( " --> in struct {}\n = use of deprecated field '_parent'\n", diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 8d0459db3..7caea1bb9 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -2,11 +2,11 @@ use std::cell::Cell; use std::str; use nom::branch::alt; -use nom::bytes::complete::{escaped, is_not, tag, take_till, take_until, take}; +use nom::bytes::complete::{escaped, is_not, tag, take_till, take_until}; use nom::character::complete::{anychar, char, digit1}; -use nom::combinator::{complete, consumed, cut, eof, map, not, opt, peek, recognize, value, fail}; +use nom::combinator::{complete, consumed, cut, eof, map, not, opt, peek, recognize, value}; use nom::error::{Error, ErrorKind}; -use nom::multi::{fold_many0, many0, many1, separated_list0, separated_list1, many_till}; +use nom::multi::{fold_many0, many0, many1, separated_list0, separated_list1}; use nom::sequence::{delimited, pair, preceded, terminated, tuple}; use nom::{self, error_position, AsChar, IResult, InputTakeAtPosition}; @@ -66,8 +66,7 @@ pub(crate) enum Expr<'a> { Call(Box>, Vec>), RustMacro(&'a str, &'a str), Try(Box>), - #[cfg(feature = "localization")] - Localize(&'a str, Option<&'a str>, Vec<(&'a str, Expr<'a>)>), + Localize(Box>, Vec<(&'a str, Expr<'a>)>), } impl Expr<'_> { @@ -670,39 +669,38 @@ macro_rules! expr_prec_layer { } } } -#[cfg(feature = "localization")] -fn quoted_ident(input: &str) -> IResult<&str, &str> { - // Match " - let (remaining, _) = tag("\"")(input)?; - let (remaining, (inner, _)) = many_till(take(1u32), tag("\""))(remaining)?; - - // Extract inner range - let length = inner.len(); - Ok((remaining, &input[1 .. length+1])) -} -#[cfg(feature = "localization")] -fn localize(i: &str) -> IResult<&str, Expr<'_>> { - let (tail, (_, _,message, attribute, args, _)) = tuple(( - tag("localize"), - ws(tag("(")), - quoted_ident, - opt(tuple((ws(tag(".")), identifier))), - opt(tuple(( - ws(tag(",")), - separated_list0(ws(tag(",")), tuple((identifier, ws(tag(":")), expr_any))), - ))), - ws(tag(")")), - ))(i)?; - Ok(( - tail, - Expr::Localize( - message, - attribute.map(|(_, a)| a), - args.map(|(_, args)| args.into_iter().map(|(k, _, v)| (k, v)).collect()) - .unwrap_or_default(), - ), - )) + +fn expr_localize_args(mut i: &str) -> IResult<&str, Vec<(&str, Expr<'_>)>> { + let mut args = Vec::<(&str, Expr<'_>)>::new(); + + let mut p = opt(tuple((ws(tag(",")), identifier, ws(tag(":")), expr_any))); + while let (j, Some((_, k, _, v))) = p(i)? { + if args.iter().any(|&(a, _)| a == k) { + eprintln!("Duplicated key: {:?}", k); + return Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))); + } + + args.push((k, v)); + i = j; + } + + let (i, _) = opt(tag(","))(i)?; + Ok((i, args)) } + +fn expr_localize(i: &str) -> IResult<&str, Expr<'_>> { + let (i, _) = pair(tag("localize"), ws(tag("(")))(i)?; + if cfg!(feature = "localization") { + cut(map( + tuple((expr_any, expr_localize_args, ws(tag(")")))), + |(text_id, args, _)| Expr::Localize(text_id.into(), args), + ))(i) + } else { + eprintln!(r#"Please activate the "localization" to use localize()."#); + Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))) + } +} + expr_prec_layer!(expr_muldivmod, expr_filtered, "*", "/", "%"); expr_prec_layer!(expr_addsub, expr_muldivmod, "+", "-"); expr_prec_layer!(expr_shifts, expr_addsub, ">>", "<<"); @@ -716,16 +714,11 @@ expr_prec_layer!(expr_or, expr_and, "||"); fn expr_handle_ws(i: &str) -> IResult<&str, Whitespace> { alt((char('-'), char('+'), char('~')))(i).map(|(s, r)| (s, Whitespace::from(r))) } + fn expr_any(i: &str) -> IResult<&str, Expr<'_>> { let range_right = |i| pair(ws(alt((tag("..="), tag("..")))), opt(expr_or))(i); alt(( - #[cfg(feature = "localization")] - map(localize, |expr| match expr { - Expr::Localize(_, _, _) => expr, - _ => panic!("localize failed: {:?}", expr), - }), - #[cfg(not(feature = "localization"))] - fail, + expr_localize, map(range_right, |(op, right)| { Expr::Range(op, None, right.map(Box::new)) }), @@ -1294,57 +1287,78 @@ mod tests { fn test_invalid_block() { super::parse("{% extend \"blah\" %}", &Syntax::default()).unwrap(); } + #[cfg(feature = "localization")] #[test] fn test_parse_localize() { + macro_rules! map { + ($($k:expr => $v:expr),* $(,)?) => {{ + use std::iter::{Iterator, IntoIterator}; + Iterator::collect(IntoIterator::into_iter([$(($k, $v),)*])) + }}; + } + assert_eq!( super::parse(r#"{{ localize("a", v: 32 + 7) }}"#, &Syntax::default()).unwrap(), vec![Node::Expr( Ws(None, None), Expr::Localize( - "a", - None, - vec![( - "v", - Expr::BinOp("+", Expr::NumLit("32").into(), Expr::NumLit("7").into()) - )] + Expr::StrLit("a").into(), + map!( + "v" => { + Expr::BinOp("+", Expr::NumLit("32").into(), Expr::NumLit("7").into()) + } + ), ) - )] + )], ); - } - #[cfg(feature = "localization")] - #[test] - fn test_parse_nested_localize() { + + assert_eq!( + super::parse( + r#"{{ localize("a", b: "b", c: "c", d: "d") }}"#, + &Syntax::default(), + ) + .unwrap(), + vec![Node::Expr( + Ws(None, None), + Expr::Localize( + Expr::StrLit("a").into(), + map!( + "b" => Expr::StrLit("b"), + "c" => Expr::StrLit("c"), + "d" => Expr::StrLit("d"), + ), + ) + )], + ); + assert_eq!( super::parse( r#"{{ localize("a", v: localize("a", v: 32 + 7) ) }}"#, - &Syntax::default() + &Syntax::default(), ) .unwrap(), vec![Node::Expr( Ws(None, None), Expr::Localize( - "a", - None, - vec![( - "v", - Expr::Localize( - "a", - None, - vec![( - "v", - Expr::BinOp( + Expr::StrLit("a").into(), + map!( + "v" => Expr::Localize( + Expr::StrLit("a").into(), + map!( + "v" => Expr::BinOp( "+", Expr::NumLit("32").into(), - Expr::NumLit("7").into() - ) - )] - ) - )] - ) - )] + Expr::NumLit("7").into(), + ), + ), + ), + ), + ), + )], ); } + #[test] fn test_parse_filter() { use Expr::*; diff --git a/testing/Cargo.toml b/testing/Cargo.toml index 84ad48bf2..d7a9d08e1 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -7,17 +7,17 @@ edition = "2018" publish = false [features] -default = ["serde-json", "markdown"] +default = ["serde-json", "markdown", "localization"] serde-json = ["serde_json", "askama/serde-json"] markdown = ["comrak", "askama/markdown"] -localization = ["fluent-templates","unic-langid"] +localization = ["fluent-templates", "unic-langid", "askama/localization"] [dependencies] askama = { path = "../askama", version = "0.11.0-beta.1" } comrak = { version = "0.13", default-features = false, optional = true } +fluent-templates = { version = "0.7.1", optional = true} serde_json = { version = "1.0", optional = true } -fluent-templates = { version= "0.7.1", optional=true} -unic-langid = {version= "0.9.0", optional=true} +unic-langid = { version = "0.9.0", optional = true } [dev-dependencies] criterion = "0.3" diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index d1577fa68..9684bb1bd 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -1,7 +1,9 @@ #![cfg(feature = "localization")] + use askama::Locale; use askama::Template; use fluent_templates::static_loader; + static_loader! { static LOCALES = { locales: "i18n-basic", @@ -11,6 +13,7 @@ static_loader! { customise: |bundle| bundle.set_use_isolating(false), }; } + #[derive(Template)] #[template(path = "i18n_invalid.html")] struct UsesI18nInvalid<'a> { @@ -27,12 +30,14 @@ struct UsesI18n<'a> { name: &'a str, hours: f64, } + #[derive(Template)] #[template(path = "i18n_no_args.html")] struct UsesNoArgsI18n<'a> { #[locale] loc: Locale<'a>, } + #[derive(Template)] #[template(path = "i18n_broken.html")] struct InvalidI18n<'a> { From 569a79b3c8ece5dde93343277153e6c77a0124b0 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Sun, 19 Jun 2022 09:02:22 +0200 Subject: [PATCH 33/57] Remove unnecessary feature guard --- askama_derive/src/input.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index 892b2b0b7..ebabfcd02 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -150,7 +150,6 @@ impl TemplateInput<'_> { mime_type, parent, path, - #[cfg(feature = "localization")] localizer, }) } From db5c3e23e7bac098d6036898e04d8b7205c5237a Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Sun, 19 Jun 2022 09:03:39 +0200 Subject: [PATCH 34/57] Corrected comment --- askama_derive/src/input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index ebabfcd02..17af58bb4 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -84,7 +84,7 @@ impl TemplateInput<'_> { match localizers.next() { Some(localizer) => { if !cfg!(feature = "localization") { - return Err("You have to active the \"localization\" feature to use #[locale] on fields.".into()); + return Err("You have to activate the \"localization\" feature to use #[locale].".into()); } else if localizers.next().is_some() { return Err("You cannot mark more than one field as #[locale].".into()); } From 129786db9e087d770ddc1b2fc04df4005087fcdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Kijewski?= Date: Sun, 19 Jun 2022 10:44:25 +0200 Subject: [PATCH 35/57] Validate localization at compile time --- askama/Cargo.toml | 7 +- askama/src/lib.rs | 60 ++++- askama_derive/Cargo.toml | 5 +- askama_derive/src/generator.rs | 2 + askama_derive/src/i18n.rs | 395 ++++++++++++++++++++++++++++ askama_derive/src/input.rs | 3 + askama_derive/src/lib.rs | 17 +- askama_derive/src/parser.rs | 48 +++- testing/Cargo.toml | 5 +- testing/i18n.toml | 12 + testing/templates/i18n_no_args.html | 2 +- testing/tests/i18n.rs | 17 +- 12 files changed, 543 insertions(+), 30 deletions(-) create mode 100644 askama_derive/src/i18n.rs create mode 100644 testing/i18n.toml diff --git a/askama/Cargo.toml b/askama/Cargo.toml index f121c3030..46dbcd3a4 100644 --- a/askama/Cargo.toml +++ b/askama/Cargo.toml @@ -19,7 +19,7 @@ maintenance = { status = "actively-developed" } default = ["config", "humansize", "num-traits", "urlencode"] config = ["askama_derive/config"] humansize = ["askama_derive/humansize", "dep_humansize"] -localization = ["askama_derive/localization", "fluent-templates", "unic-langid"] +localization = ["askama_derive/localization", "fluent-templates", "parking_lot", "unic-langid"] markdown = ["askama_derive/markdown", "comrak"] num-traits = ["askama_derive/num-traits", "dep_num_traits"] serde-json = ["askama_derive/serde-json", "askama_escape/json", "serde", "serde_json"] @@ -43,12 +43,13 @@ askama_escape = { version = "0.10.3", path = "../askama_escape" } comrak = { version = "0.13", optional = true, default-features = false } dep_humansize = { package = "humansize", version = "1.1.0", optional = true } dep_num_traits = { package = "num-traits", version = "0.2.6", optional = true } -fluent-templates = { version = "0.7.1", optional = true, default-features = false } +fluent-templates = { version = "0.7.1", optional = true } +parking_lot = { version = "0.12.1", optional = true } percent-encoding = { version = "2.1.0", optional = true } serde = { version = "1.0", optional = true, features = ["derive"] } serde_json = { version = "1.0", optional = true } serde_yaml = { version = "0.8", optional = true } -unic-langid = { version= "0.9.0", optional = true} +unic-langid = { version = "0.9.0", optional = true } [package.metadata.docs.rs] features = ["config", "humansize", "num-traits", "serde-json", "serde-yaml"] diff --git a/askama/src/lib.rs b/askama/src/lib.rs index 0fbdd69f0..1e98e4bf9 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -59,7 +59,7 @@ //! in the configuration file. The default syntax , "default", is the one //! provided by Askama. -#![forbid(unsafe_code)] +#![cfg_attr(not(feature = "localization"), forbid(unsafe))] #![deny(elided_lifetimes_in_paths)] #![deny(unreachable_pub)] @@ -69,11 +69,11 @@ pub mod helpers; use std::fmt; -pub use askama_derive::Template; +pub use askama_derive::{localization, Template}; pub use askama_escape::{Html, MarkupDisplay, Text}; #[cfg(feature = "localization")] #[doc(hidden)] -pub use fluent_templates::fluent_bundle::FluentValue; +pub use fluent_templates::{self, fluent_bundle::FluentValue}; #[cfg(feature = "localization")] use fluent_templates::{Loader, StaticLoader}; @@ -252,3 +252,57 @@ impl Locale<'_> { self.loader.lookup_complete(&self.language, text_id, args) } } + +/// Similar to OnceCell, but it has an additional take() function, which can only be used once, +/// and only if the instance was never dereferenced. +/// +/// The struct is only meant to be used by the [`localization!()`] macro. +/// Concurrent access will deliberately panic. +/// +/// Rationale: StaticLoader cannot be cloned. +#[doc(hidden)] +pub struct Unlazy(parking_lot::Mutex>); + +enum UnlazyEnum { + Generator(Option T>), + Value(Box), +} + +impl Unlazy { + pub const fn new(f: fn() -> T) -> Self { + Self(parking_lot::const_mutex(UnlazyEnum::Generator(Some(f)))) + } + + pub fn take(&self) -> T { + let f = match &mut *self.0.try_lock().unwrap() { + UnlazyEnum::Generator(f) => f.take(), + _ => None, + }; + f.unwrap()() + } +} + +impl std::ops::Deref for Unlazy +where + Self: 'static, +{ + type Target = T; + + fn deref(&self) -> &Self::Target { + let data = &mut *self.0.try_lock().unwrap(); + let value: &T = match data { + UnlazyEnum::Generator(f) => { + *data = UnlazyEnum::Value(Box::new(f.take().unwrap()())); + match data { + UnlazyEnum::Value(value) => value, + _ => unreachable!(), + } + } + UnlazyEnum::Value(value) => value, + }; + + // SAFETY: This transmutation is safe because once a value is assigned, + // it won't be unassigned again, and Self has static lifetime. + unsafe { std::mem::transmute(value) } + } +} diff --git a/askama_derive/Cargo.toml b/askama_derive/Cargo.toml index 0c7931f7d..10d900ae4 100644 --- a/askama_derive/Cargo.toml +++ b/askama_derive/Cargo.toml @@ -15,7 +15,7 @@ proc-macro = true [features] config = ["serde", "toml"] humansize = [] -localization = [] +localization = ["config", "fluent-syntax", "fluent-templates", "unic-langid"] markdown = [] num-traits = [] serde-json = [] @@ -30,6 +30,8 @@ with-tide = [] with-warp = [] [dependencies] +fluent-syntax = { version = "0.11.0", optional = true, default-features = false } +fluent-templates = { version = "0.7.1", optional = true, default-features = false } mime = "0.3" mime_guess = "2" nom = "7" @@ -38,3 +40,4 @@ quote = "1" serde = { version = "1.0", optional = true, features = ["derive"] } syn = "1" toml = { version = "0.5", optional = true } +unic-langid = { version = "0.9.0", optional = true } diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index 2fa2805c8..1c6c8c91d 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -1295,10 +1295,12 @@ impl<'a> Generator<'a> { Expr::RustMacro(name, args) => self.visit_rust_macro(buf, name, args), Expr::Try(ref expr) => self.visit_try(buf, expr.as_ref())?, Expr::Tuple(ref exprs) => self.visit_tuple(buf, exprs)?, + #[cfg(feature = "localization")] Expr::Localize(ref message, ref args) => self.visit_localize(buf, message, args)?, }) } + #[cfg(feature = "localization")] fn visit_localize( &mut self, buf: &mut Buffer, diff --git a/askama_derive/src/i18n.rs b/askama_derive/src/i18n.rs new file mode 100644 index 000000000..a124bdf86 --- /dev/null +++ b/askama_derive/src/i18n.rs @@ -0,0 +1,395 @@ +use std::collections::{HashMap, HashSet}; +use std::fmt::Display; +use std::fs::{DirEntry, OpenOptions}; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use fluent_syntax::ast::{ + Expression, InlineExpression, PatternElement, Resource, Variant, VariantKey, +}; +use fluent_syntax::parser::parse_runtime; +use fluent_templates::lazy_static::lazy_static; +use fluent_templates::loader::build_fallbacks; +use proc_macro::TokenStream; +use proc_macro2::{Ident, TokenStream as TokenStream2}; +use quote::quote_spanned; +use serde::Deserialize; +use syn::parse::{Parse, ParseStream}; +use syn::spanned::Spanned; +use syn::{parse2, Visibility}; +use toml::from_str; +use unic_langid::LanguageIdentifier; + +use crate::CompileError; + +macro_rules! mk_static { + ($(let $ident:ident: $ty:ty = $expr:expr;)*) => { + $( + let $ident = { + let value: Option<$ty> = Some($expr); + unsafe { + static mut VALUE: Option<$ty> = None; + VALUE = value; + match &VALUE { + Some(value) => value, + None => unreachable!(), + } + } + }; + )* + }; +} + +struct Variable { + vis: Visibility, + name: Ident, +} + +impl Parse for Variable { + fn parse(input: ParseStream<'_>) -> syn::Result { + let vis = input.parse().unwrap_or(Visibility::Inherited); + let name = input.parse()?; + Ok(Variable { vis, name }) + } +} + +struct Configuration { + pub(crate) fallback: LanguageIdentifier, + pub(crate) use_isolating: bool, + pub(crate) core_locales: Option<(PathBuf, Resource)>, + pub(crate) locales: Vec<(LanguageIdentifier, Vec<(PathBuf, Resource)>)>, + pub(crate) fallbacks: &'static HashMap>, + pub(crate) assets_dir: PathBuf, +} + +#[derive(Default, Deserialize)] +struct I18nConfig { + #[serde(default)] + pub(crate) fallback_language: Option, + #[serde(default)] + pub(crate) fluent: Option, +} + +#[derive(Default, Deserialize)] +struct I18nFluent { + #[serde(default)] + pub(crate) assets_dir: Option, + #[serde(default)] + pub(crate) core_locales: Option, + #[serde(default)] + pub(crate) use_isolating: Option, +} + +fn format_err(path: &Path, err: impl Display) -> String { + format!("error processing {:?}: {}", path, err) +} + +fn read_resource(path: PathBuf) -> Result<(PathBuf, Resource), String> { + let mut buf = String::new(); + OpenOptions::new() + .read(true) + .open(&path) + .map_err(|err| format_err(&path, err))? + .read_to_string(&mut buf) + .map_err(|err| format_err(&path, err))?; + + let resource = match parse_runtime(buf) { + Ok(resource) => resource, + Err((_, err_vec)) => return Err(format_err(&path, err_vec.first().unwrap())), + }; + Ok((path, resource)) +} + +fn read_lang_dir( + entry: Result, +) -> Result)>)>, String> { + let entry = match entry { + Ok(entry) => entry, + Err(_) => return Ok(None), + }; + + let language = entry + .file_name() + .to_str() + .and_then(|s| LanguageIdentifier::from_str(s).ok()); + let language: LanguageIdentifier = match language { + Some(language) => language, + None => return Ok(None), + }; + + let dir_iter = match entry.path().read_dir() { + Ok(dir_iter) => dir_iter, + Err(_) => return Ok(None), + }; + let mut resources = vec![]; + for entry in dir_iter { + if let Ok(entry) = entry { + let path = entry.path(); + if entry + .path() + .to_str() + .map(|s| s.ends_with(".ftl")) + .unwrap_or(false) + { + resources.push(read_resource(path)?); + }; + } + } + if resources.is_empty() { + return Ok(None); + } + + resources.sort_by(|(l, _), (r, _)| Path::cmp(l, r)); + Ok(Some((language, resources))) +} + +fn read_configuration() -> Result { + let root = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()); + let root = root.canonicalize().unwrap_or(root); + + let i18n_toml = root.join("i18n.toml"); + let config = match i18n_toml.exists() { + false => I18nConfig::default(), + true => { + let mut buf = String::new(); + OpenOptions::new() + .read(true) + .open(&i18n_toml) + .map_err(|err| format_err(&i18n_toml, err))? + .read_to_string(&mut buf) + .map_err(|err| format_err(&i18n_toml, err))?; + from_str(&buf).map_err(|err| format_err(&i18n_toml, err))? + } + }; + let fluent = config.fluent.unwrap_or_default(); + + let fallback = config.fallback_language.as_deref().unwrap_or("en"); + let fallback: LanguageIdentifier = match fallback.parse() { + Ok(fallback) => fallback, + Err(err) => { + return Err(format!( + "not a valid LanguageIdentifier {:?} for fallback_language: {}", + err, fallback, + ) + .into()) + } + }; + + let core_locales = match fluent.core_locales { + Some(path) => { + let path = match path.is_absolute() { + true => path, + false => root.join(path), + }; + if path.to_str().is_none() { + return Err(format!( + "core_locales path contains illegal UTF-8 characters: {:?}", + path, + )); + }; + Some(read_resource(path)?) + } + None => None, + }; + + let assets_dir = match fluent.assets_dir { + Some(path) if path.is_absolute() => todo!(), + Some(path) => root.join(&path), + None => root.join("i18n"), + }; + let mut locales = { + let mut locales = vec![]; + for entry in assets_dir + .read_dir() + .map_err(|err| format_err(&assets_dir, err))? + { + if let Some(datum) = read_lang_dir(entry)? { + locales.push(datum); + } + } + locales + }; + locales.sort_by(|(l1, _), (l2, _)| LanguageIdentifier::cmp(l1, l2)); + + mk_static! { + let locales_: Vec = locales.iter().map(|(l, _)| l.clone()).collect(); + let fallbacks: HashMap> = build_fallbacks( + &locales_, + ); + }; + + Ok(Configuration { + fallback, + use_isolating: fluent.use_isolating.unwrap_or(false), + core_locales, + locales, + fallbacks, + assets_dir, + }) +} + +fn get_i18n_config() -> Result<&'static Configuration, CompileError> { + lazy_static! { + static ref CONFIGURATION: Result = read_configuration(); + } + match &*CONFIGURATION { + Ok(configuration) => Ok(configuration), + Err(err) => Err(err.as_str().into()), + } +} + +pub(crate) fn derive(input: TokenStream) -> Result { + let configuration = get_i18n_config()?; + + let input: TokenStream2 = input.into(); + let span = input.span(); + let variable: Variable = match parse2(input) { + Ok(variable) => variable, + Err(err) => return Err(format!("could not parse localize!(…): {}", err).into()), + }; + + let vis = variable.vis; + let name = variable.name; + let assets_dir = configuration.assets_dir.to_str().unwrap(); + let fallback = configuration.fallback.to_string(); + let core_locales = configuration.core_locales.as_ref().map(|(s, _)| { + let s = s.to_str().unwrap(); + quote_spanned!(span => core_locales: #s,) + }); + let customise = match configuration.use_isolating { + false => Some(quote_spanned!(span => customise: |b| b.set_use_isolating(false),)), + true => None, + }; + + let ts = quote_spanned! { + span => + #vis static #name: + ::askama::fluent_templates::once_cell::sync::Lazy::< + ::askama::fluent_templates::StaticLoader + > = ::askama::fluent_templates::once_cell::sync::Lazy::new(|| { + mod fluent_templates { + // RATIONALE: the user might not use fluent_templates directly. + pub use ::askama::fluent_templates::*; + pub mod once_cell { + pub mod sync { + pub use ::askama::Unlazy as Lazy; + } + } + } + ::askama::fluent_templates::static_loader! { + pub static LOCALES = { + locales: #assets_dir, + fallback_language: #fallback, + #core_locales + #customise + }; + } + LOCALES.take() + }); + }; + Ok(ts.into()) +} + +pub(crate) fn arguments_of(text_id: &str) -> Result, CompileError> { + let config = get_i18n_config()?; + let entry = config.fallbacks[&config.fallback] + .iter() + .filter_map(|l1| { + config + .locales + .binary_search_by(|(l2, _)| LanguageIdentifier::cmp(l2, l1)) + .ok() + }) + .flat_map(|index| &config.locales[index].1) + .chain(config.core_locales.iter()) + .flat_map(|(_, resource)| &resource.body) + .filter_map(|entry| match entry { + fluent_syntax::ast::Entry::Message(entry) => Some(entry), + _ => None, + }) + .find_map(|entry| match entry.id.name == text_id { + true => Some(entry), + false => None, + }) + .ok_or_else(|| CompileError::from(format!("text_id {:?} not found", text_id)))?; + + let keys = entry + .value + .iter() + .flat_map(|v| v.elements.iter()) + .filter_map(|p| match p { + PatternElement::Placeable { expression } => Some(expression), + _ => None, + }) + .flat_map(expr_to_key) + .collect(); + Ok(keys) +} + +fn expr_to_key(expr: &'static Expression) -> Vec<&'static str> { + let (selector, variants): (&InlineExpression, &[Variant]) = match expr { + Expression::Select { selector, variants } => (selector, variants), + Expression::Inline(selector) => (selector, &[]), + }; + + let variant_keys = variants.iter().filter_map(|v| match &v.key { + VariantKey::Identifier { name } => Some(name.as_str()), + _ => None, + }); + + let variant_values = variants + .iter() + .flat_map(|v| v.value.elements.iter()) + .filter_map(|v| match v { + PatternElement::Placeable { expression } => Some(expression), + _ => None, + }) + .flat_map(expr_to_key); + + let selector_keys = inline_expr_to_key(selector); + + let mut v = vec![]; + v.extend(variant_keys); + v.extend(variant_values); + v.extend(selector_keys); + v +} + +fn inline_expr_to_key(selector: &'static InlineExpression) -> Vec<&'static str> { + let mut v = vec![]; + v.extend(selector_placeable(selector)); + v.extend(selector_variable(selector)); + v.extend(selector_function(selector)); + v +} + +fn selector_placeable(e: &'static InlineExpression) -> impl Iterator { + let e = match e { + InlineExpression::Placeable { expression } => Some(expression), + _ => None, + }; + e.into_iter().flat_map(|e| expr_to_key(e)) +} + +fn selector_variable(e: &'static InlineExpression) -> impl Iterator { + let id = match e { + InlineExpression::VariableReference { id } => Some(id.name.as_str()), + _ => None, + }; + id.into_iter() +} + +fn selector_function(e: &'static InlineExpression) -> impl Iterator { + let arguments = match e { + InlineExpression::FunctionReference { arguments, .. } => Some(arguments), + _ => None, + }; + arguments.into_iter().flat_map(|a| { + a.named + .iter() + .map(|n| &n.value) + .chain(&a.positional) + .flat_map(inline_expr_to_key) + }) +} diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index 17af58bb4..4eb03adbc 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -18,6 +18,7 @@ pub(crate) struct TemplateInput<'a> { pub(crate) mime_type: String, pub(crate) parent: Option<&'a syn::Type>, pub(crate) path: PathBuf, + #[cfg(feature = "localization")] pub(crate) localizer: Option, } @@ -95,6 +96,8 @@ impl TemplateInput<'_> { } _ => None, }; + #[cfg(not(feature = "localization"))] + drop(localizer); if parent.is_some() { eprint!( diff --git a/askama_derive/src/lib.rs b/askama_derive/src/lib.rs index 553da5037..ef954e8c9 100644 --- a/askama_derive/src/lib.rs +++ b/askama_derive/src/lib.rs @@ -1,4 +1,4 @@ -#![forbid(unsafe_code)] +#![cfg_attr(not(feature = "localization"), forbid(unsafe_code))] #![deny(elided_lifetimes_in_paths)] #![deny(unreachable_pub)] @@ -11,6 +11,8 @@ use proc_macro2::Span; mod config; mod generator; mod heritage; +#[cfg(feature = "localization")] +mod i18n; mod input; mod parser; @@ -19,6 +21,19 @@ pub fn derive_template(input: TokenStream) -> TokenStream { generator::derive_template(input) } +#[proc_macro] +pub fn localization(_input: TokenStream) -> TokenStream { + #[cfg(feature = "localization")] + match i18n::derive(_input) { + Ok(ts) => ts, + Err(err) => err.into_compile_error(), + } + + #[cfg(not(feature = "localization"))] + CompileError::from(r#"Activate the "localization" feature to use localization!()."#) + .into_compile_error() +} + #[derive(Debug, Clone)] struct CompileError { msg: Cow<'static, str>, diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 7caea1bb9..de9041719 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -66,6 +66,7 @@ pub(crate) enum Expr<'a> { Call(Box>, Vec>), RustMacro(&'a str, &'a str), Try(Box>), + #[cfg(feature = "localization")] Localize(Box>, Vec<(&'a str, Expr<'a>)>), } @@ -688,17 +689,48 @@ fn expr_localize_args(mut i: &str) -> IResult<&str, Vec<(&str, Expr<'_>)>> { Ok((i, args)) } +#[cfg(not(feature = "localization"))] fn expr_localize(i: &str) -> IResult<&str, Expr<'_>> { let (i, _) = pair(tag("localize"), ws(tag("(")))(i)?; - if cfg!(feature = "localization") { - cut(map( - tuple((expr_any, expr_localize_args, ws(tag(")")))), - |(text_id, args, _)| Expr::Localize(text_id.into(), args), - ))(i) - } else { - eprintln!(r#"Please activate the "localization" to use localize()."#); - Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))) + eprintln!(r#"Activate the "localization" feature to use {{ localize() }}."#); + Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))) +} + +#[cfg(feature = "localization")] +fn expr_localize(i: &str) -> IResult<&str, Expr<'_>> { + let (j, (_, _, (text_id, args, _))) = tuple(( + tag("localize"), + ws(tag("(")), + cut(tuple((expr_any, expr_localize_args, ws(tag(")"))))), + ))(i)?; + + if let Expr::StrLit(text_id) = text_id { + let mut msg_args = match crate::i18n::arguments_of(text_id) { + Ok(args) => args, + Err(err) => { + eprintln!("{}", err.msg); + return Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))); + } + }; + for &(call_arg, _) in &args { + if !msg_args.remove(call_arg) { + eprintln!( + "Fluent template {:?} does not contain argument {:?}", + text_id, call_arg, + ); + return Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))); + } + } + if !msg_args.is_empty() { + eprintln!( + "Missing argument(s) {:?} to fluent template {:?}", + msg_args, text_id, + ); + return Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))); + } } + + Ok((j, Expr::Localize(text_id.into(), args))) } expr_prec_layer!(expr_muldivmod, expr_filtered, "*", "/", "%"); diff --git a/testing/Cargo.toml b/testing/Cargo.toml index d7a9d08e1..862f41776 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -8,14 +8,13 @@ publish = false [features] default = ["serde-json", "markdown", "localization"] -serde-json = ["serde_json", "askama/serde-json"] +localization = ["unic-langid", "askama/localization"] markdown = ["comrak", "askama/markdown"] -localization = ["fluent-templates", "unic-langid", "askama/localization"] +serde-json = ["serde_json", "askama/serde-json"] [dependencies] askama = { path = "../askama", version = "0.11.0-beta.1" } comrak = { version = "0.13", default-features = false, optional = true } -fluent-templates = { version = "0.7.1", optional = true} serde_json = { version = "1.0", optional = true } unic-langid = { version = "0.9.0", optional = true } diff --git a/testing/i18n.toml b/testing/i18n.toml new file mode 100644 index 000000000..dbfa29e5e --- /dev/null +++ b/testing/i18n.toml @@ -0,0 +1,12 @@ +# Defaults to "en": +fallback_language = "en-US" + +[fluent] +# Defaults to true: +use_isolating = false + +# Defaults to "i18n": +assets_dir = "i18n-basic" + +# Default to None: +# core_locales = "…" diff --git a/testing/templates/i18n_no_args.html b/testing/templates/i18n_no_args.html index a16fadd08..c06737943 100644 --- a/testing/templates/i18n_no_args.html +++ b/testing/templates/i18n_no_args.html @@ -1 +1 @@ -

{{ localize("test", test: "") }}

\ No newline at end of file +

{{ localize("test") }}

\ No newline at end of file diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index 9684bb1bd..775438a2d 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -2,18 +2,10 @@ use askama::Locale; use askama::Template; -use fluent_templates::static_loader; -static_loader! { - static LOCALES = { - locales: "i18n-basic", - fallback_language: "en-US", - // Removes unicode isolating marks around arguments, you typically - // should only set to false when testing. - customise: |bundle| bundle.set_use_isolating(false), - }; -} +askama::localization!(LOCALES); +/* #[derive(Template)] #[template(path = "i18n_invalid.html")] struct UsesI18nInvalid<'a> { @@ -21,6 +13,7 @@ struct UsesI18nInvalid<'a> { loc: Locale<'a>, name: &'a str, } +*/ #[derive(Template)] #[template(path = "i18n.html")] @@ -38,6 +31,7 @@ struct UsesNoArgsI18n<'a> { loc: Locale<'a>, } +/* #[derive(Template)] #[template(path = "i18n_broken.html")] struct InvalidI18n<'a> { @@ -45,6 +39,7 @@ struct InvalidI18n<'a> { loc: Locale<'a>, car_color: &'a str, } +*/ #[test] fn existing_language() { @@ -82,6 +77,7 @@ fn no_args() { assert_eq!(template.render().unwrap(), r#"

This is a test

"#) } +/* #[test] fn invalid_tags_language() { let template = InvalidI18n { @@ -93,3 +89,4 @@ fn invalid_tags_language() { r#"

Unknown localization car

"# ); } +*/ From ca2ca7441be1ea904e54f90773e5724a926d52f4 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Sun, 19 Jun 2022 16:01:28 +0200 Subject: [PATCH 36/57] Fixed various Clippy complaints --- askama_derive/src/i18n.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/askama_derive/src/i18n.rs b/askama_derive/src/i18n.rs index a124bdf86..485812bf6 100644 --- a/askama_derive/src/i18n.rs +++ b/askama_derive/src/i18n.rs @@ -124,16 +124,19 @@ fn read_lang_dir( }; let mut resources = vec![]; for entry in dir_iter { - if let Ok(entry) = entry { - let path = entry.path(); - if entry - .path() - .to_str() - .map(|s| s.ends_with(".ftl")) - .unwrap_or(false) - { - resources.push(read_resource(path)?); - }; + match entry { + Ok(entry) => { + let path = entry.path(); + if entry + .path() + .to_str() + .map(|s| s.ends_with(".ftl")) + .unwrap_or(false) + { + resources.push(read_resource(path)?); + }; + } + Err(_) => continue, } } if resources.is_empty() { @@ -171,8 +174,7 @@ fn read_configuration() -> Result { return Err(format!( "not a valid LanguageIdentifier {:?} for fallback_language: {}", err, fallback, - ) - .into()) + )) } }; @@ -215,7 +217,7 @@ fn read_configuration() -> Result { mk_static! { let locales_: Vec = locales.iter().map(|(l, _)| l.clone()).collect(); let fallbacks: HashMap> = build_fallbacks( - &locales_, + locales_, ); }; From da0319b003351adf1fc46e0229d2e1b6c5845f8c Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Sun, 19 Jun 2022 19:14:42 +0200 Subject: [PATCH 37/57] added feature guards to make the compiler happy and fix errors --- askama/src/lib.rs | 8 ++++---- askama_derive/src/input.rs | 1 + askama_derive/src/parser.rs | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/askama/src/lib.rs b/askama/src/lib.rs index 1e98e4bf9..ef2a229ba 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -252,7 +252,7 @@ impl Locale<'_> { self.loader.lookup_complete(&self.language, text_id, args) } } - +#[cfg(feature = "localization")] /// Similar to OnceCell, but it has an additional take() function, which can only be used once, /// and only if the instance was never dereferenced. /// @@ -262,12 +262,12 @@ impl Locale<'_> { /// Rationale: StaticLoader cannot be cloned. #[doc(hidden)] pub struct Unlazy(parking_lot::Mutex>); - +#[cfg(feature = "localization")] enum UnlazyEnum { Generator(Option T>), Value(Box), } - +#[cfg(feature = "localization")] impl Unlazy { pub const fn new(f: fn() -> T) -> Self { Self(parking_lot::const_mutex(UnlazyEnum::Generator(Some(f)))) @@ -281,7 +281,7 @@ impl Unlazy { f.unwrap()() } } - +#[cfg(feature = "localization")] impl std::ops::Deref for Unlazy where Self: 'static, diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index 4eb03adbc..8d408a6ec 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -153,6 +153,7 @@ impl TemplateInput<'_> { mime_type, parent, path, + #[cfg(feature = "localization")] localizer, }) } diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index de9041719..78eceb882 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -670,7 +670,7 @@ macro_rules! expr_prec_layer { } } } - +#[cfg(feature = "localization")] fn expr_localize_args(mut i: &str) -> IResult<&str, Vec<(&str, Expr<'_>)>> { let mut args = Vec::<(&str, Expr<'_>)>::new(); From da9f506ac51c340bdf3727a627d422e6ee25cd39 Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Sun, 19 Jun 2022 19:19:49 +0200 Subject: [PATCH 38/57] changed forbid(unsafe) to forbid(unsafe_code) --- askama/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/askama/src/lib.rs b/askama/src/lib.rs index ef2a229ba..d217bd412 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -59,7 +59,7 @@ //! in the configuration file. The default syntax , "default", is the one //! provided by Askama. -#![cfg_attr(not(feature = "localization"), forbid(unsafe))] +#![cfg_attr(not(feature = "localization"), forbid(unsafe_code))] #![deny(elided_lifetimes_in_paths)] #![deny(unreachable_pub)] From 96c63c54c0dc594886e2c76bca7b1396e7e2366b Mon Sep 17 00:00:00 2001 From: 11tuvork28 Date: Sun, 19 Jun 2022 19:32:16 +0200 Subject: [PATCH 39/57] Created typ alias PathResources for Vec<(PathBuf, Resource --- askama_derive/src/i18n.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/askama_derive/src/i18n.rs b/askama_derive/src/i18n.rs index 485812bf6..ed031ac12 100644 --- a/askama_derive/src/i18n.rs +++ b/askama_derive/src/i18n.rs @@ -53,12 +53,12 @@ impl Parse for Variable { Ok(Variable { vis, name }) } } - +type PathResources = Vec<(PathBuf, Resource)>; struct Configuration { pub(crate) fallback: LanguageIdentifier, pub(crate) use_isolating: bool, pub(crate) core_locales: Option<(PathBuf, Resource)>, - pub(crate) locales: Vec<(LanguageIdentifier, Vec<(PathBuf, Resource)>)>, + pub(crate) locales: Vec<(LanguageIdentifier,PathResources)>, pub(crate) fallbacks: &'static HashMap>, pub(crate) assets_dir: PathBuf, } @@ -103,7 +103,7 @@ fn read_resource(path: PathBuf) -> Result<(PathBuf, Resource), String> { fn read_lang_dir( entry: Result, -) -> Result)>)>, String> { +) -> Result, String> { let entry = match entry { Ok(entry) => entry, Err(_) => return Ok(None), From 7663bf85ce4e791d68ac085ef103c1c345f0622d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Kijewski?= Date: Sun, 19 Jun 2022 19:56:18 +0200 Subject: [PATCH 40/57] Fix "all" the clippy warnings --- askama/Cargo.toml | 3 +- askama/src/i18n.rs | 84 +++++++++++++++++++++++++++++++++ askama/src/lib.rs | 92 ++----------------------------------- askama_derive/Cargo.toml | 3 +- askama_derive/src/i18n.rs | 40 ++++++---------- askama_derive/src/parser.rs | 52 ++++++++++----------- testing/Cargo.toml | 3 +- testing/tests/i18n.rs | 11 ++--- 8 files changed, 138 insertions(+), 150 deletions(-) create mode 100644 askama/src/i18n.rs diff --git a/askama/Cargo.toml b/askama/Cargo.toml index 46dbcd3a4..88bea1f62 100644 --- a/askama/Cargo.toml +++ b/askama/Cargo.toml @@ -19,7 +19,7 @@ maintenance = { status = "actively-developed" } default = ["config", "humansize", "num-traits", "urlencode"] config = ["askama_derive/config"] humansize = ["askama_derive/humansize", "dep_humansize"] -localization = ["askama_derive/localization", "fluent-templates", "parking_lot", "unic-langid"] +localization = ["askama_derive/localization", "fluent-templates", "parking_lot"] markdown = ["askama_derive/markdown", "comrak"] num-traits = ["askama_derive/num-traits", "dep_num_traits"] serde-json = ["askama_derive/serde-json", "askama_escape/json", "serde", "serde_json"] @@ -49,7 +49,6 @@ percent-encoding = { version = "2.1.0", optional = true } serde = { version = "1.0", optional = true, features = ["derive"] } serde_json = { version = "1.0", optional = true } serde_yaml = { version = "0.8", optional = true } -unic-langid = { version = "0.9.0", optional = true } [package.metadata.docs.rs] features = ["config", "humansize", "num-traits", "serde-json", "serde-yaml"] diff --git a/askama/src/i18n.rs b/askama/src/i18n.rs new file mode 100644 index 000000000..74b230862 --- /dev/null +++ b/askama/src/i18n.rs @@ -0,0 +1,84 @@ +use std::collections::HashMap; +use std::iter::FromIterator; + +use fluent_templates::fluent_bundle::FluentValue; +use fluent_templates::{LanguageIdentifier, Loader, StaticLoader}; +use parking_lot::const_mutex; + +pub struct Locale<'a> { + loader: &'a StaticLoader, + language: LanguageIdentifier, +} + +impl Locale<'_> { + pub fn new(language: LanguageIdentifier, loader: &'static StaticLoader) -> Self { + Self { loader, language } + } + + pub fn translate<'a>( + &self, + text_id: &str, + args: impl IntoIterator)>, + ) -> String { + let args = HashMap::<&str, FluentValue<'_>>::from_iter(args); + let args = match args.is_empty() { + true => None, + false => Some(&args), + }; + self.loader.lookup_complete(&self.language, text_id, args) + } +} + +/// Similar to OnceCell, but it has an additional take() function, which can only be used once, +/// and only if the instance was never dereferenced. +/// +/// The struct is only meant to be used by the [`localization!()`] macro. +/// Concurrent access will deliberately panic. +/// +/// Rationale: StaticLoader cannot be cloned. +#[doc(hidden)] +pub struct Unlazy(parking_lot::Mutex>); + +enum UnlazyEnum { + Generator(Option T>), + Value(Box), +} + +impl Unlazy { + pub const fn new(f: fn() -> T) -> Self { + Self(const_mutex(UnlazyEnum::Generator(Some(f)))) + } + + pub fn take(&self) -> T { + let f = match &mut *self.0.try_lock().unwrap() { + UnlazyEnum::Generator(f) => f.take(), + _ => None, + }; + f.unwrap()() + } +} + +impl std::ops::Deref for Unlazy +where + Self: 'static, +{ + type Target = T; + + fn deref(&self) -> &Self::Target { + let data = &mut *self.0.try_lock().unwrap(); + let value: &T = match data { + UnlazyEnum::Generator(f) => { + *data = UnlazyEnum::Value(Box::new(f.take().unwrap()())); + match data { + UnlazyEnum::Value(value) => value, + _ => unreachable!(), + } + } + UnlazyEnum::Value(value) => value, + }; + + // SAFETY: This transmutation is safe because once a value is assigned, + // it won't be unassigned again, and Self has static lifetime. + unsafe { std::mem::transmute(value) } + } +} diff --git a/askama/src/lib.rs b/askama/src/lib.rs index d217bd412..65a895d31 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -66,20 +66,21 @@ mod error; pub mod filters; pub mod helpers; +#[cfg(feature = "localization")] +mod i18n; use std::fmt; pub use askama_derive::{localization, Template}; pub use askama_escape::{Html, MarkupDisplay, Text}; #[cfg(feature = "localization")] -#[doc(hidden)] -pub use fluent_templates::{self, fluent_bundle::FluentValue}; -#[cfg(feature = "localization")] -use fluent_templates::{Loader, StaticLoader}; +pub use fluent_templates::{self, fluent_bundle::FluentValue, fs::langid, LanguageIdentifier}; #[doc(hidden)] pub use crate as shared; pub use crate::error::{Error, Result}; +#[cfg(feature = "localization")] +pub use crate::i18n::{Locale, Unlazy}; /// Main `Template` trait; implementations are generally derived /// @@ -223,86 +224,3 @@ mod tests { note = "file-level dependency tracking is handled automatically without build script" )] pub fn rerun_if_templates_changed() {} - -#[cfg(feature = "localization")] -pub struct Locale<'a> { - loader: &'a StaticLoader, - language: unic_langid::LanguageIdentifier, -} - -#[cfg(feature = "localization")] -impl Locale<'_> { - pub fn new(language: unic_langid::LanguageIdentifier, loader: &'static StaticLoader) -> Self { - Self { loader, language } - } - - pub fn translate<'a>( - &self, - text_id: &str, - args: impl IntoIterator)>, - ) -> String { - use std::collections::HashMap; - use std::iter::FromIterator; - - let args = HashMap::<&str, FluentValue<'_>>::from_iter(args); - let args = match args.is_empty() { - true => None, - false => Some(&args), - }; - self.loader.lookup_complete(&self.language, text_id, args) - } -} -#[cfg(feature = "localization")] -/// Similar to OnceCell, but it has an additional take() function, which can only be used once, -/// and only if the instance was never dereferenced. -/// -/// The struct is only meant to be used by the [`localization!()`] macro. -/// Concurrent access will deliberately panic. -/// -/// Rationale: StaticLoader cannot be cloned. -#[doc(hidden)] -pub struct Unlazy(parking_lot::Mutex>); -#[cfg(feature = "localization")] -enum UnlazyEnum { - Generator(Option T>), - Value(Box), -} -#[cfg(feature = "localization")] -impl Unlazy { - pub const fn new(f: fn() -> T) -> Self { - Self(parking_lot::const_mutex(UnlazyEnum::Generator(Some(f)))) - } - - pub fn take(&self) -> T { - let f = match &mut *self.0.try_lock().unwrap() { - UnlazyEnum::Generator(f) => f.take(), - _ => None, - }; - f.unwrap()() - } -} -#[cfg(feature = "localization")] -impl std::ops::Deref for Unlazy -where - Self: 'static, -{ - type Target = T; - - fn deref(&self) -> &Self::Target { - let data = &mut *self.0.try_lock().unwrap(); - let value: &T = match data { - UnlazyEnum::Generator(f) => { - *data = UnlazyEnum::Value(Box::new(f.take().unwrap()())); - match data { - UnlazyEnum::Value(value) => value, - _ => unreachable!(), - } - } - UnlazyEnum::Value(value) => value, - }; - - // SAFETY: This transmutation is safe because once a value is assigned, - // it won't be unassigned again, and Self has static lifetime. - unsafe { std::mem::transmute(value) } - } -} diff --git a/askama_derive/Cargo.toml b/askama_derive/Cargo.toml index 10d900ae4..ec0422093 100644 --- a/askama_derive/Cargo.toml +++ b/askama_derive/Cargo.toml @@ -15,7 +15,7 @@ proc-macro = true [features] config = ["serde", "toml"] humansize = [] -localization = ["config", "fluent-syntax", "fluent-templates", "unic-langid"] +localization = ["fluent-syntax", "fluent-templates", "serde", "toml"] markdown = [] num-traits = [] serde-json = [] @@ -40,4 +40,3 @@ quote = "1" serde = { version = "1.0", optional = true, features = ["derive"] } syn = "1" toml = { version = "0.5", optional = true } -unic-langid = { version = "0.9.0", optional = true } diff --git a/askama_derive/src/i18n.rs b/askama_derive/src/i18n.rs index ed031ac12..72c6a8cfc 100644 --- a/askama_derive/src/i18n.rs +++ b/askama_derive/src/i18n.rs @@ -11,6 +11,7 @@ use fluent_syntax::ast::{ use fluent_syntax::parser::parse_runtime; use fluent_templates::lazy_static::lazy_static; use fluent_templates::loader::build_fallbacks; +use fluent_templates::LanguageIdentifier; use proc_macro::TokenStream; use proc_macro2::{Ident, TokenStream as TokenStream2}; use quote::quote_spanned; @@ -19,10 +20,11 @@ use syn::parse::{Parse, ParseStream}; use syn::spanned::Spanned; use syn::{parse2, Visibility}; use toml::from_str; -use unic_langid::LanguageIdentifier; use crate::CompileError; +type FileResource = (PathBuf, Resource); + macro_rules! mk_static { ($(let $ident:ident: $ty:ty = $expr:expr;)*) => { $( @@ -53,12 +55,12 @@ impl Parse for Variable { Ok(Variable { vis, name }) } } -type PathResources = Vec<(PathBuf, Resource)>; + struct Configuration { pub(crate) fallback: LanguageIdentifier, pub(crate) use_isolating: bool, - pub(crate) core_locales: Option<(PathBuf, Resource)>, - pub(crate) locales: Vec<(LanguageIdentifier,PathResources)>, + pub(crate) core_locales: Option, + pub(crate) locales: Vec<(LanguageIdentifier, Vec)>, pub(crate) fallbacks: &'static HashMap>, pub(crate) assets_dir: PathBuf, } @@ -85,7 +87,7 @@ fn format_err(path: &Path, err: impl Display) -> String { format!("error processing {:?}: {}", path, err) } -fn read_resource(path: PathBuf) -> Result<(PathBuf, Resource), String> { +fn read_resource(path: PathBuf) -> Result { let mut buf = String::new(); OpenOptions::new() .read(true) @@ -103,7 +105,7 @@ fn read_resource(path: PathBuf) -> Result<(PathBuf, Resource), String> { fn read_lang_dir( entry: Result, -) -> Result, String> { +) -> Result)>, String> { let entry = match entry { Ok(entry) => entry, Err(_) => return Ok(None), @@ -122,22 +124,13 @@ fn read_lang_dir( Ok(dir_iter) => dir_iter, Err(_) => return Ok(None), }; + let mut resources = vec![]; - for entry in dir_iter { - match entry { - Ok(entry) => { - let path = entry.path(); - if entry - .path() - .to_str() - .map(|s| s.ends_with(".ftl")) - .unwrap_or(false) - { - resources.push(read_resource(path)?); - }; - } - Err(_) => continue, - } + for entry in dir_iter.flatten() { + let path = entry.path(); + if path.to_str().map(|s| s.ends_with(".ftl")).unwrap_or(false) { + resources.push(read_resource(path)?); + }; } if resources.is_empty() { return Ok(None); @@ -310,10 +303,7 @@ pub(crate) fn arguments_of(text_id: &str) -> Result, Compi fluent_syntax::ast::Entry::Message(entry) => Some(entry), _ => None, }) - .find_map(|entry| match entry.id.name == text_id { - true => Some(entry), - false => None, - }) + .find(|entry| entry.id.name == text_id) .ok_or_else(|| CompileError::from(format!("text_id {:?} not found", text_id)))?; let keys = entry diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 78eceb882..0e75b46a8 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -670,24 +670,6 @@ macro_rules! expr_prec_layer { } } } -#[cfg(feature = "localization")] -fn expr_localize_args(mut i: &str) -> IResult<&str, Vec<(&str, Expr<'_>)>> { - let mut args = Vec::<(&str, Expr<'_>)>::new(); - - let mut p = opt(tuple((ws(tag(",")), identifier, ws(tag(":")), expr_any))); - while let (j, Some((_, k, _, v))) = p(i)? { - if args.iter().any(|&(a, _)| a == k) { - eprintln!("Duplicated key: {:?}", k); - return Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))); - } - - args.push((k, v)); - i = j; - } - - let (i, _) = opt(tag(","))(i)?; - Ok((i, args)) -} #[cfg(not(feature = "localization"))] fn expr_localize(i: &str) -> IResult<&str, Expr<'_>> { @@ -698,10 +680,28 @@ fn expr_localize(i: &str) -> IResult<&str, Expr<'_>> { #[cfg(feature = "localization")] fn expr_localize(i: &str) -> IResult<&str, Expr<'_>> { + fn localize_args(mut i: &str) -> IResult<&str, Vec<(&str, Expr<'_>)>> { + let mut args = Vec::<(&str, Expr<'_>)>::new(); + + let mut p = opt(tuple((ws(tag(",")), identifier, ws(tag(":")), expr_any))); + while let (j, Some((_, k, _, v))) = p(i)? { + if args.iter().any(|&(a, _)| a == k) { + eprintln!("Duplicated key: {:?}", k); + return Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))); + } + + args.push((k, v)); + i = j; + } + + let (i, _) = opt(tag(","))(i)?; + Ok((i, args)) + } + let (j, (_, _, (text_id, args, _))) = tuple(( tag("localize"), ws(tag("(")), - cut(tuple((expr_any, expr_localize_args, ws(tag(")"))))), + cut(tuple((expr_any, localize_args, ws(tag(")"))))), ))(i)?; if let Expr::StrLit(text_id) = text_id { @@ -1331,11 +1331,11 @@ mod tests { } assert_eq!( - super::parse(r#"{{ localize("a", v: 32 + 7) }}"#, &Syntax::default()).unwrap(), + super::parse(r#"{{ localize(1, v: 32 + 7) }}"#, &Syntax::default()).unwrap(), vec![Node::Expr( Ws(None, None), Expr::Localize( - Expr::StrLit("a").into(), + Expr::NumLit("1").into(), map!( "v" => { Expr::BinOp("+", Expr::NumLit("32").into(), Expr::NumLit("7").into()) @@ -1347,14 +1347,14 @@ mod tests { assert_eq!( super::parse( - r#"{{ localize("a", b: "b", c: "c", d: "d") }}"#, + r#"{{ localize(1, b: "b", c: "c", d: "d") }}"#, &Syntax::default(), ) .unwrap(), vec![Node::Expr( Ws(None, None), Expr::Localize( - Expr::StrLit("a").into(), + Expr::NumLit("1").into(), map!( "b" => Expr::StrLit("b"), "c" => Expr::StrLit("c"), @@ -1366,17 +1366,17 @@ mod tests { assert_eq!( super::parse( - r#"{{ localize("a", v: localize("a", v: 32 + 7) ) }}"#, + r#"{{ localize(1, v: localize(2, v: 32 + 7) ) }}"#, &Syntax::default(), ) .unwrap(), vec![Node::Expr( Ws(None, None), Expr::Localize( - Expr::StrLit("a").into(), + Expr::NumLit("1").into(), map!( "v" => Expr::Localize( - Expr::StrLit("a").into(), + Expr::NumLit("2").into(), map!( "v" => Expr::BinOp( "+", diff --git a/testing/Cargo.toml b/testing/Cargo.toml index 862f41776..1b463e2db 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -8,7 +8,7 @@ publish = false [features] default = ["serde-json", "markdown", "localization"] -localization = ["unic-langid", "askama/localization"] +localization = ["askama/localization"] markdown = ["comrak", "askama/markdown"] serde-json = ["serde_json", "askama/serde-json"] @@ -16,7 +16,6 @@ serde-json = ["serde_json", "askama/serde-json"] askama = { path = "../askama", version = "0.11.0-beta.1" } comrak = { version = "0.13", default-features = false, optional = true } serde_json = { version = "1.0", optional = true } -unic-langid = { version = "0.9.0", optional = true } [dev-dependencies] criterion = "0.3" diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index 775438a2d..2a85e5699 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -1,7 +1,6 @@ #![cfg(feature = "localization")] -use askama::Locale; -use askama::Template; +use askama::{langid, Locale, Template}; askama::localization!(LOCALES); @@ -44,7 +43,7 @@ struct InvalidI18n<'a> { #[test] fn existing_language() { let template = UsesI18n { - loc: Locale::new(unic_langid::langid!("es-MX"), &LOCALES), + loc: Locale::new(langid!("es-MX"), &LOCALES), name: "Hilda", hours: 300072.3, }; @@ -58,7 +57,7 @@ fn existing_language() { #[test] fn fallback_language() { let template = UsesI18n { - loc: Locale::new(unic_langid::langid!("nl-BE"), &LOCALES), + loc: Locale::new(langid!("nl-BE"), &LOCALES), name: "Hilda", hours: 300072.3, }; @@ -72,7 +71,7 @@ fn fallback_language() { #[test] fn no_args() { let template = UsesNoArgsI18n { - loc: Locale::new(unic_langid::langid!("en-US"), &LOCALES), + loc: Locale::new(langid!("en-US"), &LOCALES), }; assert_eq!(template.render().unwrap(), r#"

This is a test

"#) } @@ -81,7 +80,7 @@ fn no_args() { #[test] fn invalid_tags_language() { let template = InvalidI18n { - loc: Locale::new(unic_langid::langid!("nl-BE"), &LOCALES), + loc: Locale::new(langid!("nl-BE"), &LOCALES), car_color: "Red", }; assert_eq!( From 6872a725418793b5e11e3f91a1e1a047ff64d95c Mon Sep 17 00:00:00 2001 From: 11Tuvork28 Date: Mon, 5 Sep 2022 18:31:46 +0200 Subject: [PATCH 41/57] Fix lint job --- askama_derive/src/input.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index 05661cf4b..b07416f1a 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -47,10 +47,10 @@ impl TemplateInput<'_> { (&Source::Path(ref path), _) => config.find_template(path, None)?, (&Source::Source(_), Some(ext)) => PathBuf::from(format!("{}.{}", ast.ident, ext)), (&Source::Source(_), None) => { - return Err("must include 'ext' attribute when using 'source' attribute".into()) + return Err("must include 'ext' attribute when using 'source' attribute".into()) } }; - + let localizer = match ast.data { syn::Data::Struct(syn::DataStruct { fields: syn::Fields::Named(ref fields), From 92fadc9bba0c607c48078a7b22d6e64d9da0051c Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Mon, 10 Oct 2022 16:41:11 +0200 Subject: [PATCH 42/57] fix: Remove commented out i18n tests Fixes thread https://github.com/djc/askama/pull/700/files#r934558288 --- testing/templates/i18n_broken.html | 1 - testing/templates/i18n_invalid.html | 2 -- testing/tests/i18n.rs | 34 ----------------------------- 3 files changed, 37 deletions(-) delete mode 100644 testing/templates/i18n_broken.html delete mode 100644 testing/templates/i18n_invalid.html diff --git a/testing/templates/i18n_broken.html b/testing/templates/i18n_broken.html deleted file mode 100644 index 983f484e1..000000000 --- a/testing/templates/i18n_broken.html +++ /dev/null @@ -1 +0,0 @@ -

{{ localize("car", color: car_color) }}

\ No newline at end of file diff --git a/testing/templates/i18n_invalid.html b/testing/templates/i18n_invalid.html deleted file mode 100644 index f5afefa58..000000000 --- a/testing/templates/i18n_invalid.html +++ /dev/null @@ -1,2 +0,0 @@ -

{{ localize("greetingsss", name: name) }}

-

{{ localize("ages") }}

\ No newline at end of file diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index 2a85e5699..3dac96a79 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -4,16 +4,6 @@ use askama::{langid, Locale, Template}; askama::localization!(LOCALES); -/* -#[derive(Template)] -#[template(path = "i18n_invalid.html")] -struct UsesI18nInvalid<'a> { - #[locale] - loc: Locale<'a>, - name: &'a str, -} -*/ - #[derive(Template)] #[template(path = "i18n.html")] struct UsesI18n<'a> { @@ -30,16 +20,6 @@ struct UsesNoArgsI18n<'a> { loc: Locale<'a>, } -/* -#[derive(Template)] -#[template(path = "i18n_broken.html")] -struct InvalidI18n<'a> { - #[locale] - loc: Locale<'a>, - car_color: &'a str, -} -*/ - #[test] fn existing_language() { let template = UsesI18n { @@ -75,17 +55,3 @@ fn no_args() { }; assert_eq!(template.render().unwrap(), r#"

This is a test

"#) } - -/* -#[test] -fn invalid_tags_language() { - let template = InvalidI18n { - loc: Locale::new(langid!("nl-BE"), &LOCALES), - car_color: "Red", - }; - assert_eq!( - template.render().unwrap(), - r#"

Unknown localization car

"# - ); -} -*/ From 98a47777ae340f723e935a24f74eb992b5af9bf7 Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Mon, 10 Oct 2022 16:45:04 +0200 Subject: [PATCH 43/57] fix: Add trailing newline to i18n test templates --- testing/templates/i18n.html | 2 +- testing/templates/i18n_no_args.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/templates/i18n.html b/testing/templates/i18n.html index 76f116ad6..990c1eaeb 100644 --- a/testing/templates/i18n.html +++ b/testing/templates/i18n.html @@ -1,2 +1,2 @@

{{ localize("greeting", name: name) }}

-

{{ localize("age", hours: hours ) }}

\ No newline at end of file +

{{ localize("age", hours: hours ) }}

diff --git a/testing/templates/i18n_no_args.html b/testing/templates/i18n_no_args.html index c06737943..e4863bddc 100644 --- a/testing/templates/i18n_no_args.html +++ b/testing/templates/i18n_no_args.html @@ -1 +1 @@ -

{{ localize("test") }}

\ No newline at end of file +

{{ localize("test") }}

From 63ed5c93d7f9841bf2d54c45e0cbbec126d155d6 Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Mon, 10 Oct 2022 16:47:08 +0200 Subject: [PATCH 44/57] fix: `cargo fmt`, remove trailing whitespace --- askama_derive/src/input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index b07416f1a..defdd1c9a 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -47,7 +47,7 @@ impl TemplateInput<'_> { (&Source::Path(ref path), _) => config.find_template(path, None)?, (&Source::Source(_), Some(ext)) => PathBuf::from(format!("{}.{}", ast.ident, ext)), (&Source::Source(_), None) => { - return Err("must include 'ext' attribute when using 'source' attribute".into()) + return Err("must include 'ext' attribute when using 'source' attribute".into()) } }; From 073e92a7fa1f44d44d6a138cfdb1551a313d389d Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Mon, 10 Oct 2022 17:03:18 +0200 Subject: [PATCH 45/57] fix: Rename feature `localization` to `i18n` --- askama/Cargo.toml | 2 +- askama/src/lib.rs | 8 ++++---- askama_derive/Cargo.toml | 2 +- askama_derive/src/generator.rs | 4 ++-- askama_derive/src/input.rs | 13 ++++++++----- askama_derive/src/lib.rs | 11 +++++------ askama_derive/src/parser.rs | 10 +++++----- testing/Cargo.toml | 4 ++-- testing/tests/i18n.rs | 2 +- 9 files changed, 29 insertions(+), 27 deletions(-) diff --git a/askama/Cargo.toml b/askama/Cargo.toml index 4eb88324e..797ec4250 100644 --- a/askama/Cargo.toml +++ b/askama/Cargo.toml @@ -19,7 +19,7 @@ maintenance = { status = "actively-developed" } default = ["config", "humansize", "num-traits", "urlencode"] config = ["askama_derive/config"] humansize = ["askama_derive/humansize", "dep_humansize"] -localization = ["askama_derive/localization", "fluent-templates", "parking_lot"] +i18n = ["askama_derive/i18n", "fluent-templates", "parking_lot"] markdown = ["askama_derive/markdown", "comrak"] num-traits = ["askama_derive/num-traits", "dep_num_traits"] serde-json = ["askama_derive/serde-json", "askama_escape/json", "serde", "serde_json"] diff --git a/askama/src/lib.rs b/askama/src/lib.rs index 65a895d31..e2353a785 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -59,27 +59,27 @@ //! in the configuration file. The default syntax , "default", is the one //! provided by Askama. -#![cfg_attr(not(feature = "localization"), forbid(unsafe_code))] +#![cfg_attr(not(feature = "i18n"), forbid(unsafe_code))] #![deny(elided_lifetimes_in_paths)] #![deny(unreachable_pub)] mod error; pub mod filters; pub mod helpers; -#[cfg(feature = "localization")] +#[cfg(feature = "i18n")] mod i18n; use std::fmt; pub use askama_derive::{localization, Template}; pub use askama_escape::{Html, MarkupDisplay, Text}; -#[cfg(feature = "localization")] +#[cfg(feature = "i18n")] pub use fluent_templates::{self, fluent_bundle::FluentValue, fs::langid, LanguageIdentifier}; #[doc(hidden)] pub use crate as shared; pub use crate::error::{Error, Result}; -#[cfg(feature = "localization")] +#[cfg(feature = "i18n")] pub use crate::i18n::{Locale, Unlazy}; /// Main `Template` trait; implementations are generally derived diff --git a/askama_derive/Cargo.toml b/askama_derive/Cargo.toml index 3b5cc3cd9..67b68c0e9 100644 --- a/askama_derive/Cargo.toml +++ b/askama_derive/Cargo.toml @@ -15,7 +15,7 @@ proc-macro = true [features] config = ["serde", "toml"] humansize = [] -localization = ["fluent-syntax", "fluent-templates", "serde", "toml"] +i18n = ["fluent-syntax", "fluent-templates", "serde", "toml"] markdown = [] num-traits = [] serde-json = [] diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index 2194f18e4..da10a3750 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -1299,12 +1299,12 @@ impl<'a> Generator<'a> { Expr::RustMacro(name, args) => self.visit_rust_macro(buf, name, args), Expr::Try(ref expr) => self.visit_try(buf, expr.as_ref())?, Expr::Tuple(ref exprs) => self.visit_tuple(buf, exprs)?, - #[cfg(feature = "localization")] + #[cfg(feature = "i18n")] Expr::Localize(ref message, ref args) => self.visit_localize(buf, message, args)?, }) } - #[cfg(feature = "localization")] + #[cfg(feature = "i18n")] fn visit_localize( &mut self, buf: &mut Buffer, diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index defdd1c9a..90d73bd01 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -17,7 +17,7 @@ pub(crate) struct TemplateInput<'a> { pub(crate) ext: Option, pub(crate) mime_type: String, pub(crate) path: PathBuf, - #[cfg(feature = "localization")] + #[cfg(feature = "i18n")] pub(crate) localizer: Option, } @@ -69,8 +69,11 @@ impl TemplateInput<'_> { ); match localizers.next() { Some(localizer) => { - if !cfg!(feature = "localization") { - return Err("You have to activate the \"localization\" feature to use #[locale].".into()); + if !cfg!(feature = "i18n") { + return Err( + "You have to activate the \"i18n\" feature to use #[locale]." + .into(), + ); } else if localizers.next().is_some() { return Err("You cannot mark more than one field as #[locale].".into()); } @@ -81,7 +84,7 @@ impl TemplateInput<'_> { } _ => None, }; - #[cfg(not(feature = "localization"))] + #[cfg(not(feature = "i18n"))] drop(localizer); // Validate syntax @@ -130,7 +133,7 @@ impl TemplateInput<'_> { ext, mime_type, path, - #[cfg(feature = "localization")] + #[cfg(feature = "i18n")] localizer, }) } diff --git a/askama_derive/src/lib.rs b/askama_derive/src/lib.rs index ef954e8c9..243a2f586 100644 --- a/askama_derive/src/lib.rs +++ b/askama_derive/src/lib.rs @@ -1,4 +1,4 @@ -#![cfg_attr(not(feature = "localization"), forbid(unsafe_code))] +#![cfg_attr(not(feature = "i18n"), forbid(unsafe_code))] #![deny(elided_lifetimes_in_paths)] #![deny(unreachable_pub)] @@ -11,7 +11,7 @@ use proc_macro2::Span; mod config; mod generator; mod heritage; -#[cfg(feature = "localization")] +#[cfg(feature = "i18n")] mod i18n; mod input; mod parser; @@ -23,15 +23,14 @@ pub fn derive_template(input: TokenStream) -> TokenStream { #[proc_macro] pub fn localization(_input: TokenStream) -> TokenStream { - #[cfg(feature = "localization")] + #[cfg(feature = "i18n")] match i18n::derive(_input) { Ok(ts) => ts, Err(err) => err.into_compile_error(), } - #[cfg(not(feature = "localization"))] - CompileError::from(r#"Activate the "localization" feature to use localization!()."#) - .into_compile_error() + #[cfg(not(feature = "i18n"))] + CompileError::from(r#"Activate the "i18n" feature to use localization!()."#).into_compile_error() } #[derive(Debug, Clone)] diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 0e75b46a8..589b28179 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -66,7 +66,7 @@ pub(crate) enum Expr<'a> { Call(Box>, Vec>), RustMacro(&'a str, &'a str), Try(Box>), - #[cfg(feature = "localization")] + #[cfg(feature = "i18n")] Localize(Box>, Vec<(&'a str, Expr<'a>)>), } @@ -671,14 +671,14 @@ macro_rules! expr_prec_layer { } } -#[cfg(not(feature = "localization"))] +#[cfg(not(feature = "i18n"))] fn expr_localize(i: &str) -> IResult<&str, Expr<'_>> { let (i, _) = pair(tag("localize"), ws(tag("(")))(i)?; - eprintln!(r#"Activate the "localization" feature to use {{ localize() }}."#); + eprintln!(r#"Activate the "i18n" feature to use {{ localize() }}."#); Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))) } -#[cfg(feature = "localization")] +#[cfg(feature = "i18n")] fn expr_localize(i: &str) -> IResult<&str, Expr<'_>> { fn localize_args(mut i: &str) -> IResult<&str, Vec<(&str, Expr<'_>)>> { let mut args = Vec::<(&str, Expr<'_>)>::new(); @@ -1320,7 +1320,7 @@ mod tests { super::parse("{% extend \"blah\" %}", &Syntax::default()).unwrap(); } - #[cfg(feature = "localization")] + #[cfg(feature = "i18n")] #[test] fn test_parse_localize() { macro_rules! map { diff --git a/testing/Cargo.toml b/testing/Cargo.toml index ccdc813eb..fcec61cf0 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -7,8 +7,8 @@ edition = "2018" publish = false [features] -default = ["serde-json", "markdown", "localization"] -localization = ["askama/localization"] +default = ["serde-json", "markdown", "i18n"] +i18n = ["askama/i18n"] markdown = ["comrak", "askama/markdown"] serde-json = ["serde_json", "askama/serde-json"] diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index 3dac96a79..7e67ac384 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -1,4 +1,4 @@ -#![cfg(feature = "localization")] +#![cfg(feature = "i18n")] use askama::{langid, Locale, Template}; From f944c868b366a5d33ff3c43f8482a6164bb1470f Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Mon, 10 Oct 2022 17:15:02 +0200 Subject: [PATCH 46/57] opinionated: Rename initialization macro `localization!` to `i18n_load!` --- askama/src/i18n.rs | 2 +- askama/src/lib.rs | 2 +- askama_derive/src/i18n.rs | 2 +- askama_derive/src/lib.rs | 4 ++-- testing/tests/i18n.rs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/askama/src/i18n.rs b/askama/src/i18n.rs index 74b230862..2fe1ee6de 100644 --- a/askama/src/i18n.rs +++ b/askama/src/i18n.rs @@ -32,7 +32,7 @@ impl Locale<'_> { /// Similar to OnceCell, but it has an additional take() function, which can only be used once, /// and only if the instance was never dereferenced. /// -/// The struct is only meant to be used by the [`localization!()`] macro. +/// The struct is only meant to be used by the [`i18n_load!()`] macro. /// Concurrent access will deliberately panic. /// /// Rationale: StaticLoader cannot be cloned. diff --git a/askama/src/lib.rs b/askama/src/lib.rs index e2353a785..e77a71b76 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -71,7 +71,7 @@ mod i18n; use std::fmt; -pub use askama_derive::{localization, Template}; +pub use askama_derive::{i18n_load, Template}; pub use askama_escape::{Html, MarkupDisplay, Text}; #[cfg(feature = "i18n")] pub use fluent_templates::{self, fluent_bundle::FluentValue, fs::langid, LanguageIdentifier}; diff --git a/askama_derive/src/i18n.rs b/askama_derive/src/i18n.rs index 72c6a8cfc..5fe045199 100644 --- a/askama_derive/src/i18n.rs +++ b/askama_derive/src/i18n.rs @@ -241,7 +241,7 @@ pub(crate) fn derive(input: TokenStream) -> Result { let span = input.span(); let variable: Variable = match parse2(input) { Ok(variable) => variable, - Err(err) => return Err(format!("could not parse localize!(…): {}", err).into()), + Err(err) => return Err(format!("could not parse i18n_load!(…): {}", err).into()), }; let vis = variable.vis; diff --git a/askama_derive/src/lib.rs b/askama_derive/src/lib.rs index 243a2f586..c1f47e119 100644 --- a/askama_derive/src/lib.rs +++ b/askama_derive/src/lib.rs @@ -22,7 +22,7 @@ pub fn derive_template(input: TokenStream) -> TokenStream { } #[proc_macro] -pub fn localization(_input: TokenStream) -> TokenStream { +pub fn i18n_load(_input: TokenStream) -> TokenStream { #[cfg(feature = "i18n")] match i18n::derive(_input) { Ok(ts) => ts, @@ -30,7 +30,7 @@ pub fn localization(_input: TokenStream) -> TokenStream { } #[cfg(not(feature = "i18n"))] - CompileError::from(r#"Activate the "i18n" feature to use localization!()."#).into_compile_error() + CompileError::from(r#"Activate the "i18n" feature to use i18n_load!()."#).into_compile_error() } #[derive(Debug, Clone)] diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index 7e67ac384..cd2215ad8 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -2,7 +2,7 @@ use askama::{langid, Locale, Template}; -askama::localization!(LOCALES); +askama::i18n_load!(LOCALES); #[derive(Template)] #[template(path = "i18n.html")] From c32b22d043db92728d6d2d2493fa91d434befe9b Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Mon, 10 Oct 2022 17:49:35 +0200 Subject: [PATCH 47/57] style: Change wording "have to" to "need to" --- askama_derive/src/generator.rs | 2 +- askama_derive/src/input.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index da10a3750..61066ea6e 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -1313,7 +1313,7 @@ impl<'a> Generator<'a> { ) -> Result { let localizer = self.input.localizer.as_deref().ok_or( - "You have to annotate a field with #[locale] to use the localize() function.", + "You need to annotate a field with #[locale] to use the localize() function.", )?; buf.write(&format!( diff --git a/askama_derive/src/input.rs b/askama_derive/src/input.rs index 90d73bd01..cb5bb6cdb 100644 --- a/askama_derive/src/input.rs +++ b/askama_derive/src/input.rs @@ -71,7 +71,7 @@ impl TemplateInput<'_> { Some(localizer) => { if !cfg!(feature = "i18n") { return Err( - "You have to activate the \"i18n\" feature to use #[locale]." + "You need to activate the \"i18n\" feature to use #[locale]." .into(), ); } else if localizers.next().is_some() { From 426575e493faba5eb4ae5d27d6cc2323f86d526d Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Mon, 10 Oct 2022 20:04:07 +0200 Subject: [PATCH 48/57] refactor: Confine i18n code inside i18n modules Proc-macro askama_derive::i18n_load!() must remain in root namespace, due to restrictions in proc-macro exports. However, it is re-published as `askama::i18n::load!()`. --- askama/src/i18n.rs | 9 +++++++-- askama/src/lib.rs | 8 ++------ askama_derive/src/generator.rs | 2 +- askama_derive/src/i18n.rs | 14 +++++++------- askama_derive/src/lib.rs | 2 +- testing/Cargo.toml | 3 ++- testing/tests/i18n.rs | 5 +++-- 7 files changed, 23 insertions(+), 20 deletions(-) diff --git a/askama/src/i18n.rs b/askama/src/i18n.rs index 2fe1ee6de..3b1a13742 100644 --- a/askama/src/i18n.rs +++ b/askama/src/i18n.rs @@ -1,8 +1,13 @@ use std::collections::HashMap; use std::iter::FromIterator; -use fluent_templates::fluent_bundle::FluentValue; -use fluent_templates::{LanguageIdentifier, Loader, StaticLoader}; +/// Re-export conventiently as `askama::i18n::load!()`. +/// Proc-macro crates can only export macros from their root namespace. +#[doc(hidden)] +pub use askama_derive::i18n_load as load; + +pub use fluent_templates::{self, fluent_bundle::FluentValue, fs::langid, LanguageIdentifier}; +use fluent_templates::{Loader, StaticLoader}; use parking_lot::const_mutex; pub struct Locale<'a> { diff --git a/askama/src/lib.rs b/askama/src/lib.rs index e77a71b76..a7947c944 100644 --- a/askama/src/lib.rs +++ b/askama/src/lib.rs @@ -67,20 +67,16 @@ mod error; pub mod filters; pub mod helpers; #[cfg(feature = "i18n")] -mod i18n; +pub mod i18n; use std::fmt; -pub use askama_derive::{i18n_load, Template}; +pub use askama_derive::Template; pub use askama_escape::{Html, MarkupDisplay, Text}; -#[cfg(feature = "i18n")] -pub use fluent_templates::{self, fluent_bundle::FluentValue, fs::langid, LanguageIdentifier}; #[doc(hidden)] pub use crate as shared; pub use crate::error::{Error, Result}; -#[cfg(feature = "i18n")] -pub use crate::i18n::{Locale, Unlazy}; /// Main `Template` trait; implementations are generally derived /// diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index 61066ea6e..898ff8e2c 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -1324,7 +1324,7 @@ impl<'a> Generator<'a> { buf.writeln(", [")?; buf.indent(); for (k, v) in args { - buf.write(&format!("({:?}, ::askama::FluentValue::from(", k)); + buf.write(&format!("({:?}, ::askama::i18n::FluentValue::from(", k)); self.visit_expr(buf, v)?; buf.writeln(")),")?; } diff --git a/askama_derive/src/i18n.rs b/askama_derive/src/i18n.rs index 5fe045199..44cc90dfe 100644 --- a/askama_derive/src/i18n.rs +++ b/askama_derive/src/i18n.rs @@ -234,7 +234,7 @@ fn get_i18n_config() -> Result<&'static Configuration, CompileError> { } } -pub(crate) fn derive(input: TokenStream) -> Result { +pub(crate) fn load(input: TokenStream) -> Result { let configuration = get_i18n_config()?; let input: TokenStream2 = input.into(); @@ -260,19 +260,19 @@ pub(crate) fn derive(input: TokenStream) -> Result { let ts = quote_spanned! { span => #vis static #name: - ::askama::fluent_templates::once_cell::sync::Lazy::< - ::askama::fluent_templates::StaticLoader - > = ::askama::fluent_templates::once_cell::sync::Lazy::new(|| { + ::fluent_templates::once_cell::sync::Lazy::< + ::fluent_templates::StaticLoader + > = ::fluent_templates::once_cell::sync::Lazy::new(|| { mod fluent_templates { // RATIONALE: the user might not use fluent_templates directly. - pub use ::askama::fluent_templates::*; + pub use ::fluent_templates::*; pub mod once_cell { pub mod sync { - pub use ::askama::Unlazy as Lazy; + pub use ::askama::i18n::Unlazy as Lazy; } } } - ::askama::fluent_templates::static_loader! { + ::fluent_templates::static_loader! { pub static LOCALES = { locales: #assets_dir, fallback_language: #fallback, diff --git a/askama_derive/src/lib.rs b/askama_derive/src/lib.rs index c1f47e119..35deda7fa 100644 --- a/askama_derive/src/lib.rs +++ b/askama_derive/src/lib.rs @@ -24,7 +24,7 @@ pub fn derive_template(input: TokenStream) -> TokenStream { #[proc_macro] pub fn i18n_load(_input: TokenStream) -> TokenStream { #[cfg(feature = "i18n")] - match i18n::derive(_input) { + match i18n::load(_input) { Ok(ts) => ts, Err(err) => err.into_compile_error(), } diff --git a/testing/Cargo.toml b/testing/Cargo.toml index fcec61cf0..7207d2993 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -8,13 +8,14 @@ publish = false [features] default = ["serde-json", "markdown", "i18n"] -i18n = ["askama/i18n"] +i18n = ["askama/i18n", "fluent-templates"] markdown = ["comrak", "askama/markdown"] serde-json = ["serde_json", "askama/serde-json"] [dependencies] askama = { path = "../askama", version = "0.11.0-beta.1" } comrak = { version = "0.14", default-features = false, optional = true } +fluent-templates = { version = "0.7.1", optional = true, default-features = false } serde_json = { version = "1.0", optional = true } [dev-dependencies] diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index cd2215ad8..ff43506ba 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -1,8 +1,9 @@ #![cfg(feature = "i18n")] -use askama::{langid, Locale, Template}; +use askama::i18n::{langid, Locale}; +use askama::Template; -askama::i18n_load!(LOCALES); +askama::i18n::load!(LOCALES); #[derive(Template)] #[template(path = "i18n.html")] From 81cce1a395c81342aacbab2db687ec3ba6ea57e7 Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Mon, 10 Oct 2022 20:05:36 +0200 Subject: [PATCH 49/57] fix: Hide dependency `fluent_templates` from library users --- askama_derive/src/i18n.rs | 10 +++++----- testing/Cargo.toml | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/askama_derive/src/i18n.rs b/askama_derive/src/i18n.rs index 44cc90dfe..966a0804a 100644 --- a/askama_derive/src/i18n.rs +++ b/askama_derive/src/i18n.rs @@ -260,19 +260,19 @@ pub(crate) fn load(input: TokenStream) -> Result { let ts = quote_spanned! { span => #vis static #name: - ::fluent_templates::once_cell::sync::Lazy::< - ::fluent_templates::StaticLoader - > = ::fluent_templates::once_cell::sync::Lazy::new(|| { + ::askama::i18n::fluent_templates::once_cell::sync::Lazy::< + ::askama::i18n::fluent_templates::StaticLoader + > = ::askama::i18n::fluent_templates::once_cell::sync::Lazy::new(|| { mod fluent_templates { // RATIONALE: the user might not use fluent_templates directly. - pub use ::fluent_templates::*; + pub use ::askama::i18n::fluent_templates::*; pub mod once_cell { pub mod sync { pub use ::askama::i18n::Unlazy as Lazy; } } } - ::fluent_templates::static_loader! { + ::askama::i18n::fluent_templates::static_loader! { pub static LOCALES = { locales: #assets_dir, fallback_language: #fallback, diff --git a/testing/Cargo.toml b/testing/Cargo.toml index 7207d2993..fcec61cf0 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -8,14 +8,13 @@ publish = false [features] default = ["serde-json", "markdown", "i18n"] -i18n = ["askama/i18n", "fluent-templates"] +i18n = ["askama/i18n"] markdown = ["comrak", "askama/markdown"] serde-json = ["serde_json", "askama/serde-json"] [dependencies] askama = { path = "../askama", version = "0.11.0-beta.1" } comrak = { version = "0.14", default-features = false, optional = true } -fluent-templates = { version = "0.7.1", optional = true, default-features = false } serde_json = { version = "1.0", optional = true } [dev-dependencies] From 1c315d6a34f97cca3c7915fb6833c845f592b6ec Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Mon, 10 Oct 2022 20:19:06 +0200 Subject: [PATCH 50/57] fix(deps): Update dependency `fluent-templates` to `0.8.0` --- askama/Cargo.toml | 2 +- askama/src/i18n.rs | 2 +- askama_derive/Cargo.toml | 2 +- askama_derive/src/generator.rs | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/askama/Cargo.toml b/askama/Cargo.toml index 797ec4250..9c6f8b21a 100644 --- a/askama/Cargo.toml +++ b/askama/Cargo.toml @@ -44,7 +44,7 @@ askama_escape = { version = "0.10.3", path = "../askama_escape" } comrak = { version = "0.14", optional = true, default-features = false } dep_humansize = { package = "humansize", version = "1.1.0", optional = true } dep_num_traits = { package = "num-traits", version = "0.2.6", optional = true } -fluent-templates = { version = "0.7.1", optional = true } +fluent-templates = { version = "0.8.0", optional = true } parking_lot = { version = "0.12.1", optional = true } percent-encoding = { version = "2.1.0", optional = true } serde = { version = "1.0", optional = true, features = ["derive"] } diff --git a/askama/src/i18n.rs b/askama/src/i18n.rs index 3b1a13742..f640b0b30 100644 --- a/askama/src/i18n.rs +++ b/askama/src/i18n.rs @@ -24,7 +24,7 @@ impl Locale<'_> { &self, text_id: &str, args: impl IntoIterator)>, - ) -> String { + ) -> Option { let args = HashMap::<&str, FluentValue<'_>>::from_iter(args); let args = match args.is_empty() { true => None, diff --git a/askama_derive/Cargo.toml b/askama_derive/Cargo.toml index 67b68c0e9..c4c5805e2 100644 --- a/askama_derive/Cargo.toml +++ b/askama_derive/Cargo.toml @@ -32,7 +32,7 @@ with-warp = [] [dependencies] fluent-syntax = { version = "0.11.0", optional = true, default-features = false } -fluent-templates = { version = "0.7.1", optional = true, default-features = false } +fluent-templates = { version = "0.8.0", optional = true, default-features = false } mime = "0.3" mime_guess = "2" nom = "7" diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index 898ff8e2c..7d4f68fb2 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -1329,7 +1329,8 @@ impl<'a> Generator<'a> { buf.writeln(")),")?; } buf.dedent()?; - buf.write("])"); + // Safe to unwrap, as `message` is checked at compile time. + buf.write("]).unwrap()"); Ok(DisplayWrap::Unwrapped) } From 6384ae23bfa481e986bf178625f6920b36959bb0 Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Mon, 10 Oct 2022 21:01:00 +0200 Subject: [PATCH 51/57] docs: Add initial i18n module documentation --- askama/src/i18n.rs | 46 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/askama/src/i18n.rs b/askama/src/i18n.rs index f640b0b30..37d3700cf 100644 --- a/askama/src/i18n.rs +++ b/askama/src/i18n.rs @@ -1,9 +1,49 @@ +//! Module for compile time checked localization +//! +//! # Example: +//! +//! [Fluent Translation List](https://projectfluent.org/) resource file `i18n/es-MX/basic.ftl`: +//! +//! ```ftl +//! greeting = ¡Hola, { $name }! +//! ``` +//! +//! Askama HTML template `templates/example.html`: +//! +//! ```html +//!

{{ localize("greeting", name: name) }}

+//! ``` +//! +//! Rust usage: +//! ```ignore +//! use askama::i18n::{langid, Locale}; +//! use askama::Template; +//! +//! askama::i18n::load!(LOCALES); +//! +//! #[derive(Template)] +//! #[template(path = "example.html")] +//! struct UsesI18n<'a> { +//! #[locale] +//! loc: Locale<'a>, +//! name: &'a str, +//! } +//! +//! let template = UsesI18n { +//! loc: Locale::new(langid!("es-MX"), &LOCALES), +//! name: "Hilda", +//! }; +//! +//! // "

¡Hola, Hilda!

" +//! template.render().unwrap(); +//! ``` + use std::collections::HashMap; use std::iter::FromIterator; -/// Re-export conventiently as `askama::i18n::load!()`. -/// Proc-macro crates can only export macros from their root namespace. -#[doc(hidden)] +// Re-export conventiently as `askama::i18n::load!()`. +// Proc-macro crates can only export macros from their root namespace. +/// Load locales at compile time. See example above for usage. pub use askama_derive::i18n_load as load; pub use fluent_templates::{self, fluent_bundle::FluentValue, fs::langid, LanguageIdentifier}; From 3f002ef033891cf71b7f8416d0f9ba538a60f079 Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Tue, 11 Oct 2022 18:27:03 +0200 Subject: [PATCH 52/57] docs: Rename example template struct --- askama/src/i18n.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/askama/src/i18n.rs b/askama/src/i18n.rs index 37d3700cf..3a9ee9135 100644 --- a/askama/src/i18n.rs +++ b/askama/src/i18n.rs @@ -23,13 +23,13 @@ //! //! #[derive(Template)] //! #[template(path = "example.html")] -//! struct UsesI18n<'a> { +//! struct ExampleTemplate<'a> { //! #[locale] //! loc: Locale<'a>, //! name: &'a str, //! } //! -//! let template = UsesI18n { +//! let template = ExampleTemplate { //! loc: Locale::new(langid!("es-MX"), &LOCALES), //! name: "Hilda", //! }; From b52aaf30f180170f884b541e20597d5d606122c9 Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Tue, 11 Oct 2022 19:39:05 +0200 Subject: [PATCH 53/57] fix: Implement missing recursive `is_cachable()` for `Expr::Localize` --- askama_derive/src/parser.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 1c76c20e7..6998b7148 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -135,6 +135,9 @@ impl Expr<'_> { } Expr::Group(arg) => arg.is_cachable(), Expr::Tuple(args) => args.iter().all(|arg| arg.is_cachable()), + Expr::Localize(text_id, args) => { + text_id.is_cachable() && args.iter().all(|(_, arg)| arg.is_cachable()) + } // We have too little information to tell if the expression is pure: Expr::Call(_, _) => false, Expr::RustMacro(_, _) => false, From 9909c36cbe106e6b5feae29838d49926a0e077d0 Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Tue, 11 Oct 2022 19:58:28 +0200 Subject: [PATCH 54/57] fix(style): Name localization message identifier as in Fluent Project Quote from https://projectfluent.org/fluent/guide/hello.html ```ftl hello = Hello, world! ``` Each message has an identifier that allows the developer to bind it to the place in the software where it will be used. The above message is called `hello`. --- askama/src/i18n.rs | 4 ++-- askama_derive/src/generator.rs | 8 ++++---- askama_derive/src/i18n.rs | 6 +++--- askama_derive/src/parser.rs | 16 ++++++++-------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/askama/src/i18n.rs b/askama/src/i18n.rs index 3a9ee9135..1a77aabe3 100644 --- a/askama/src/i18n.rs +++ b/askama/src/i18n.rs @@ -62,7 +62,7 @@ impl Locale<'_> { pub fn translate<'a>( &self, - text_id: &str, + msg_id: &str, args: impl IntoIterator)>, ) -> Option { let args = HashMap::<&str, FluentValue<'_>>::from_iter(args); @@ -70,7 +70,7 @@ impl Locale<'_> { true => None, false => Some(&args), }; - self.loader.lookup_complete(&self.language, text_id, args) + self.loader.lookup_complete(&self.language, msg_id, args) } } diff --git a/askama_derive/src/generator.rs b/askama_derive/src/generator.rs index 0b905231d..ac7ed92d8 100644 --- a/askama_derive/src/generator.rs +++ b/askama_derive/src/generator.rs @@ -1292,7 +1292,7 @@ impl<'a> Generator<'a> { Expr::Try(ref expr) => self.visit_try(buf, expr.as_ref())?, Expr::Tuple(ref exprs) => self.visit_tuple(buf, exprs)?, #[cfg(feature = "i18n")] - Expr::Localize(ref message, ref args) => self.visit_localize(buf, message, args)?, + Expr::Localize(ref msg_id, ref args) => self.visit_localize(buf, msg_id, args)?, }) } @@ -1300,7 +1300,7 @@ impl<'a> Generator<'a> { fn visit_localize( &mut self, buf: &mut Buffer, - message: &Expr<'_>, + msg_id: &Expr<'_>, args: &[(&str, Expr<'_>)], ) -> Result { let localizer = @@ -1312,7 +1312,7 @@ impl<'a> Generator<'a> { "self.{}.translate(", normalize_identifier(localizer) )); - self.visit_expr(buf, message)?; + self.visit_expr(buf, msg_id)?; buf.writeln(", [")?; buf.indent(); for (k, v) in args { @@ -1321,7 +1321,7 @@ impl<'a> Generator<'a> { buf.writeln(")),")?; } buf.dedent()?; - // Safe to unwrap, as `message` is checked at compile time. + // Safe to unwrap, as `msg_id` is checked at compile time. buf.write("]).unwrap()"); Ok(DisplayWrap::Unwrapped) diff --git a/askama_derive/src/i18n.rs b/askama_derive/src/i18n.rs index 966a0804a..2f80a1131 100644 --- a/askama_derive/src/i18n.rs +++ b/askama_derive/src/i18n.rs @@ -286,7 +286,7 @@ pub(crate) fn load(input: TokenStream) -> Result { Ok(ts.into()) } -pub(crate) fn arguments_of(text_id: &str) -> Result, CompileError> { +pub(crate) fn arguments_of(msg_id: &str) -> Result, CompileError> { let config = get_i18n_config()?; let entry = config.fallbacks[&config.fallback] .iter() @@ -303,8 +303,8 @@ pub(crate) fn arguments_of(text_id: &str) -> Result, Compi fluent_syntax::ast::Entry::Message(entry) => Some(entry), _ => None, }) - .find(|entry| entry.id.name == text_id) - .ok_or_else(|| CompileError::from(format!("text_id {:?} not found", text_id)))?; + .find(|entry| entry.id.name == msg_id) + .ok_or_else(|| CompileError::from(format!("msg_id {:?} not found", msg_id)))?; let keys = entry .value diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 6998b7148..6f61876be 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -135,8 +135,8 @@ impl Expr<'_> { } Expr::Group(arg) => arg.is_cachable(), Expr::Tuple(args) => args.iter().all(|arg| arg.is_cachable()), - Expr::Localize(text_id, args) => { - text_id.is_cachable() && args.iter().all(|(_, arg)| arg.is_cachable()) + Expr::Localize(msg_id, args) => { + msg_id.is_cachable() && args.iter().all(|(_, arg)| arg.is_cachable()) } // We have too little information to tell if the expression is pure: Expr::Call(_, _) => false, @@ -734,14 +734,14 @@ fn expr_localize(i: &str) -> IResult<&str, Expr<'_>> { Ok((i, args)) } - let (j, (_, _, (text_id, args, _))) = tuple(( + let (j, (_, _, (msg_id, args, _))) = tuple(( tag("localize"), ws(tag("(")), cut(tuple((expr_any, localize_args, ws(tag(")"))))), ))(i)?; - if let Expr::StrLit(text_id) = text_id { - let mut msg_args = match crate::i18n::arguments_of(text_id) { + if let Expr::StrLit(msg_id) = msg_id { + let mut msg_args = match crate::i18n::arguments_of(msg_id) { Ok(args) => args, Err(err) => { eprintln!("{}", err.msg); @@ -752,7 +752,7 @@ fn expr_localize(i: &str) -> IResult<&str, Expr<'_>> { if !msg_args.remove(call_arg) { eprintln!( "Fluent template {:?} does not contain argument {:?}", - text_id, call_arg, + msg_id, call_arg, ); return Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))); } @@ -760,13 +760,13 @@ fn expr_localize(i: &str) -> IResult<&str, Expr<'_>> { if !msg_args.is_empty() { eprintln!( "Missing argument(s) {:?} to fluent template {:?}", - msg_args, text_id, + msg_args, msg_id, ); return Err(nom::Err::Failure(error_position!(i, ErrorKind::Tag))); } } - Ok((j, Expr::Localize(text_id.into(), args))) + Ok((j, Expr::Localize(msg_id.into(), args))) } expr_prec_layer!(expr_muldivmod, expr_filtered, "*", "/", "%"); From 6cda9f57b8c522d083b14af4b4e5fd1bc4d20842 Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Tue, 11 Oct 2022 20:07:56 +0200 Subject: [PATCH 55/57] fix(style): Normalize test names, prefix all with `test_` --- testing/tests/i18n.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/tests/i18n.rs b/testing/tests/i18n.rs index ff43506ba..f1240e3aa 100644 --- a/testing/tests/i18n.rs +++ b/testing/tests/i18n.rs @@ -22,7 +22,7 @@ struct UsesNoArgsI18n<'a> { } #[test] -fn existing_language() { +fn test_existing_language() { let template = UsesI18n { loc: Locale::new(langid!("es-MX"), &LOCALES), name: "Hilda", @@ -36,7 +36,7 @@ fn existing_language() { } #[test] -fn fallback_language() { +fn test_fallback_language() { let template = UsesI18n { loc: Locale::new(langid!("nl-BE"), &LOCALES), name: "Hilda", @@ -50,7 +50,7 @@ fn fallback_language() { } #[test] -fn no_args() { +fn test_no_args() { let template = UsesNoArgsI18n { loc: Locale::new(langid!("en-US"), &LOCALES), }; From 7c159041f1da5936d0d96f0b8d93275c51da4a2b Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Tue, 11 Oct 2022 20:23:01 +0200 Subject: [PATCH 56/57] fix: Add missing trailing newline --- testing/i18n-basic/en-US/basic.ftl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/i18n-basic/en-US/basic.ftl b/testing/i18n-basic/en-US/basic.ftl index eb94ab9c7..c1cc18561 100644 --- a/testing/i18n-basic/en-US/basic.ftl +++ b/testing/i18n-basic/en-US/basic.ftl @@ -1,3 +1,3 @@ -greeting = Hello, { $name }! +greeting = Hello, { $name }! age = You are { $hours } hours old. -test = This is a test \ No newline at end of file +test = This is a test From 2bbba70d27a213cdaaa2645f5bc354cb2c871a9a Mon Sep 17 00:00:00 2001 From: LeoniePhiline <22329650+LeoniePhiline@users.noreply.github.com> Date: Tue, 11 Oct 2022 21:00:58 +0200 Subject: [PATCH 57/57] fix: Compilation without feature `i18n` failed --- askama_derive/src/parser.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/askama_derive/src/parser.rs b/askama_derive/src/parser.rs index 6f61876be..de704d801 100644 --- a/askama_derive/src/parser.rs +++ b/askama_derive/src/parser.rs @@ -135,6 +135,7 @@ impl Expr<'_> { } Expr::Group(arg) => arg.is_cachable(), Expr::Tuple(args) => args.iter().all(|arg| arg.is_cachable()), + #[cfg(feature = "i18n")] Expr::Localize(msg_id, args) => { msg_id.is_cachable() && args.iter().all(|(_, arg)| arg.is_cachable()) }