Add whitespace stripping in the input

In HTML sources indentation can aid readability, but it only bloats the
output. This PR adds the argument "strip" to templates, which lets you
automatically strip the template input:

* "none": Don't strip any spaces in the input
* "tail": Remove a single single newline at the end of the input. This
  is the default
* "trim-lines": Remove all whitespaces at the front and back of all
  lines, and remove empty lines
* "eager": Like "trim", but also replace runs of whitespaces with a
  single space.

The automatic stripping does not try to understand the input at all. It
does not know what `xml:space="preserve"`, `<pre>` or `<textarea>`
means.

askama_derive/src/lib.rs

@@ -37,8 +37,8 @@ fn build_template(ast: &syn::DeriveInput) -> Result<String, CompileError> {
     let config = Config::new(&config_toml)?;
     let input = TemplateInput::new(ast, &config)?;
     let source: String = match input.source {
-        Source::Source(ref s) => s.clone(),
-        Source::Path(_) => get_template_source(&input.path)?,
+        Source::Source(ref s) => input.strip.apply(s),
+        Source::Path(_) => get_template_source(&input.path, input.strip)?,
     };
 
     let mut sources = HashMap::new();
@@ -95,12 +95,12 @@ fn find_used_templates(
                         )));
                     }
                     dependency_graph.push(dependency_path);
-                    let source = get_template_source(&extends)?;
+                    let source = get_template_source(&extends, input.strip)?;
                     check.push((extends, source));
                 }
                 Node::Import(_, import, _) => {
                     let import = input.config.find_template(import, Some(&path))?;
-                    let source = get_template_source(&import)?;
+                    let source = get_template_source(&import, input.strip)?;
                     check.push((import, source));
                 }
                 _ => {}

askama_shared/src/generator.rs

@@ -1,11 +1,9 @@
-use super::{get_template_source, CompileError, Integrations};
-use crate::filters;
 use crate::heritage::{Context, Heritage};
 use crate::input::{Source, TemplateInput};
 use crate::parser::{parse, Cond, CondTest, Expr, Loop, Node, Target, When, Ws};
+use crate::{filters, get_template_source, CompileError, Integrations};
 
 use proc_macro2::Span;
-
 use quote::{quote, ToTokens};
 
 use std::collections::HashMap;
@@ -765,7 +763,7 @@ impl<'a, S: std::hash::BuildHasher> Generator<'a, S> {
             .input
             .config
             .find_template(path, Some(&self.input.path))?;
-        let src = get_template_source(&path)?;
+        let src = get_template_source(&path, self.input.strip)?;
         let nodes = parse(&src, self.input.syntax)?;
 
         // Make sure the compiler understands that the generated code depends on the template file.

askama_shared/src/input.rs

@@ -1,4 +1,4 @@
-use crate::{CompileError, Config, Syntax};
+use crate::{CompileError, Config, Strip, Syntax};
 
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
@@ -17,6 +17,7 @@ pub struct TemplateInput<'a> {
     pub mime_type: String,
     pub parent: Option<&'a syn::Type>,
     pub path: PathBuf,
+    pub strip: Strip,
 }
 
 impl TemplateInput<'_> {
@@ -57,6 +58,7 @@ impl TemplateInput<'_> {
         let mut escaping = None;
         let mut ext = None;
         let mut syntax = None;
+        let mut strip = config.strip;
         for item in template_args {
             let pair = match item {
                 syn::NestedMeta::Meta(syn::Meta::NameValue(ref pair)) => pair,
@@ -111,6 +113,12 @@ impl TemplateInput<'_> {
                 } else {
                     return Err("syntax value must be string literal".into());
                 }
+            } else if pair.path.is_ident("strip") {
+                if let syn::Lit::Str(ref s) = pair.lit {
+                    strip = s.value().parse()?;
+                } else {
+                    return Err("syntax strip must be string literal".into());
+                }
             } else {
                 return Err(format!(
                     "unsupported attribute key '{}' found",
@@ -199,6 +207,7 @@ impl TemplateInput<'_> {
             mime_type,
             parent,
             path,
+            strip,
         })
     }
 

askama_shared/src/lib.rs

@@ -12,6 +12,7 @@ use std::{env, fmt, fs};
 use serde::Deserialize;
 
 pub use crate::input::extension_to_mime_type;
+pub use crate::strip::Strip;
 pub use askama_escape::MarkupDisplay;
 
 mod error;
@@ -26,13 +27,15 @@ pub mod heritage;
 pub mod input;
 #[doc(hidden)]
 pub mod parser;
+mod strip;
 
 #[derive(Debug)]
 pub struct Config<'a> {
     pub dirs: Vec<PathBuf>,
     pub syntaxes: BTreeMap<String, Syntax<'a>>,
     pub default_syntax: &'a str,
     pub escapers: Vec<(HashSet<String>, String)>,
+    pub strip: Strip,
 }
 
 impl Config<'_> {
@@ -101,6 +104,7 @@ impl Config<'_> {
             syntaxes,
             default_syntax,
             escapers,
+            strip: raw.strip.unwrap_or_default(),
         })
     }
 
@@ -199,6 +203,7 @@ struct RawConfig<'d> {
     general: Option<General<'d>>,
     syntax: Option<Vec<RawSyntax<'d>>>,
     escaper: Option<Vec<RawEscaper<'d>>>,
+    strip: Option<Strip>,
 }
 
 impl RawConfig<'_> {
@@ -259,18 +264,16 @@ where
 }
 
 #[allow(clippy::match_wild_err_arm)]
-pub fn get_template_source(tpl_path: &Path) -> std::result::Result<String, CompileError> {
+pub fn get_template_source(
+    tpl_path: &Path,
+    strip: Strip,
+) -> std::result::Result<String, CompileError> {
     match fs::read_to_string(tpl_path) {
         Err(_) => Err(CompileError::String(format!(
             "unable to open template file '{}'",
             tpl_path.to_str().unwrap()
         ))),
-        Ok(mut source) => {
-            if source.ends_with('\n') {
-                let _ = source.pop();
-            }
-            Ok(source)
-        }
+        Ok(source) => Ok(strip.apply(source)),
     }
 }
 
@@ -332,7 +335,7 @@ mod tests {
         let path = Config::new("")
             .and_then(|config| config.find_template("b.html", None))
             .unwrap();
-        assert_eq!(get_template_source(&path).unwrap(), "bar");
+        assert_eq!(get_template_source(&path, Strip::Tail).unwrap(), "bar");
     }
 
     #[test]

askama_shared/src/strip.rs

@@ -0,0 +1,117 @@
+use crate::CompileError;
+
+use std::borrow::Cow;
+use std::convert::{TryFrom, TryInto};
+use std::str::FromStr;
+
+/// Whitespace handling of the input source.
+///
+/// The automatic stripping does not try to understand the input at all. It
+/// does not know what `xml:space="preserve"`, `<pre>` or `<textarea>`
+/// means.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Strip {
+    /// "none": Don't strip any spaces in the input
+    None,
+    /// "tail": Remove a single single newline at the end of the input. This is the default
+    Tail,
+    /// "trim-lines": Remove all whitespaces at the front and back of all lines, and remove empty
+    /// lines
+    TrimLines,
+    /// "eager": Like "trim", but also replace runs of whitespaces with a single space.
+    Eager,
+}
+
+impl Default for Strip {
+    fn default() -> Self {
+        Strip::Tail
+    }
+}
+
+impl TryFrom<&str> for Strip {
+    type Error = String;
+
+    fn try_from(s: &str) -> Result<Self, Self::Error> {
+        match s {
+            "none" => Ok(Strip::None),
+            "tail" => Ok(Strip::Tail),
+            "trim-lines" => Ok(Strip::TrimLines),
+            "eager" => Ok(Strip::Eager),
+            v => return Err(format!("invalid value for strip: {:?}", v)),
+        }
+    }
+}
+
+impl FromStr for Strip {
+    type Err = CompileError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        s.try_into().map_err(Into::into)
+    }
+}
+
+#[cfg(feature = "serde")]
+const _: () = {
+    use std::fmt;
+
+    struct StripVisitor;
+
+    impl<'de> serde::de::Visitor<'de> for StripVisitor {
+        type Value = Strip;
+
+        fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            write!(f, r#"the string "none", "tail", "trim-lines", or "eager""#)
+        }
+
+        fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>
+        where
+            E: serde::de::Error,
+        {
+            s.try_into().map_err(E::custom)
+        }
+    }
+
+    impl<'de> serde::Deserialize<'de> for Strip {
+        fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+        where
+            D: serde::Deserializer<'de>,
+        {
+            deserializer.deserialize_str(StripVisitor)
+        }
+    }
+};
+
+impl Strip {
+    pub fn apply<'a, S: Into<Cow<'a, str>>>(self, src: S) -> String {
+        let src = src.into();
+        match self {
+            Strip::None => src.into_owned(),
+            Strip::Tail => {
+                let mut s = src.into_owned();
+                if s.ends_with('\n') {
+                    s.pop();
+                }
+                s
+            }
+            Strip::TrimLines | Strip::Eager => {
+                let mut stripped = String::with_capacity(src.len());
+                for line in src.lines().map(|s| s.trim()).filter(|&s| !s.is_empty()) {
+                    if !stripped.is_empty() {
+                        stripped.push('\n');
+                    }
+                    if self == Strip::Eager {
+                        for (index, word) in line.split_ascii_whitespace().enumerate() {
+                            if index > 0 {
+                                stripped.push(' ');
+                            }
+                            stripped.push_str(word);
+                        }
+                    } else {
+                        stripped.push_str(line);
+                    }
+                }
+                stripped
+            }
+        }
+    }
+}

testing/templates/whitespace_trimming.html

@@ -0,0 +1,10 @@
+
+<!DOCTYPE html>
+
+<html>
+ <body>
+  <p>
+   .  .   .
+  </p>
+ </body>
+</html>

testing/tests/whitespace.rs

@@ -39,3 +39,91 @@ fn test_extra_whitespace() {
     template.nested_1.nested_2.hash.insert("key", "value");
     assert_eq!(template.render().unwrap(), "\n0\n0\n0\n0\n\n\n\n0\n0\n0\n0\n0\n\na0\na1\nvalue\n\n\n\n\n\n[\n  \"a0\",\n  \"a1\",\n  \"a2\",\n  \"a3\"\n]\n[\n  \"a0\",\n  \"a1\",\n  \"a2\",\n  \"a3\"\n][\n  \"a0\",\n  \"a1\",\n  \"a2\",\n  \"a3\"\n]\n[\n  \"a1\"\n][\n  \"a1\"\n]\n[\n  \"a1\",\n  \"a2\"\n][\n  \"a1\",\n  \"a2\"\n]\n[\n  \"a1\"\n][\n  \"a1\"\n]1-1-1\n3333 3\n2222 2\n0000 0\n3333 3\n\ntruefalse\nfalsefalsefalse\n\n\n\n\n\n\n\n\n\n\n\n\n\n");
 }
+
+#[derive(askama::Template)]
+#[template(source = " a \t b   c\n\nd e\n  f\n\n\n", ext = "txt", strip = "none")]
+struct StripNone;
+
+#[test]
+fn test_strip_none() {
+    assert_eq!(StripNone.render().unwrap(), " a \t b   c\n\nd e\n  f");
+}
+
+#[derive(askama::Template)]
+#[template(source = " a \t b   c\n\nd e\n  f\n\n\n", ext = "txt", strip = "tail")]
+struct StripTail;
+
+#[test]
+fn test_strip_tail() {
+    assert_eq!(StripTail.render().unwrap(), " a \t b   c\n\nd e\n  f");
+}
+
+#[derive(askama::Template)]
+#[template(
+    source = " a \t b   c\n\nd e\n  f\n\n\n",
+    ext = "txt",
+    strip = "trim-lines"
+)]
+struct StripTrimLines;
+
+#[test]
+fn test_strip_trim_lines() {
+    assert_eq!(StripTrimLines.render().unwrap(), "a \t b   c\nd e\nf");
+}
+
+#[derive(askama::Template)]
+#[template(source = " a \t b   c\n\nd e\n  f\n\n\n", ext = "txt", strip = "eager")]
+struct StripEager;
+
+#[test]
+fn test_strip_eager() {
+    assert_eq!(StripEager.render().unwrap(), "a b c\nd e\nf");
+}
+
+#[derive(askama::Template)]
+#[template(path = "whitespace_trimming.html", strip = "none")]
+struct StripNone2;
+
+#[test]
+fn test_strip_none2() {
+    assert_eq!(
+        StripNone2.render().unwrap(),
+        "\n<!DOCTYPE html>\n\n<html>\n <body>\n  <p>\n   .  .   .\n  </p>\n </body>\n</html>"
+    );
+}
+
+#[derive(askama::Template)]
+#[template(path = "whitespace_trimming.html", strip = "tail")]
+struct StripTail2;
+
+#[test]
+fn test_strip_tail2() {
+    assert_eq!(
+        StripTail2.render().unwrap(),
+        "\n<!DOCTYPE html>\n\n<html>\n <body>\n  <p>\n   .  .   .\n  </p>\n </body>\n</html>"
+    );
+}
+
+#[derive(askama::Template)]
+#[template(path = "whitespace_trimming.html", strip = "trim-lines")]
+struct StripTrimLines2;
+
+#[test]
+fn test_strip_trim_lines2() {
+    assert_eq!(
+        StripTrimLines2.render().unwrap(),
+        "<!DOCTYPE html>\n<html>\n<body>\n<p>\n.  .   .\n</p>\n</body>\n</html>"
+    );
+}
+
+#[derive(askama::Template)]
+#[template(path = "whitespace_trimming.html", strip = "eager")]
+struct StripEager2;
+
+#[test]
+fn test_strip_eager2() {
+    assert_eq!(
+        StripEager2.render().unwrap(),
+        "<!DOCTYPE html>\n<html>\n<body>\n<p>\n. . .\n</p>\n</body>\n</html>"
+    );
+}