Add support for auto-indented blocks

Fixes pallets#178

Blocks now support a new syntax '{%* ... %}' that alings the indentation of
multiline string with the block statement itself. This is especially
useful with YAML or other languages where indentation matter. Example:

labels.j2:
```
tla: webtool
env: {{ env }}
```

deployment.yaml.j2:
```
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    {% include 'labels.j2' %}
  name: webtool
spec:
  selector:
    matchLabels:
      {% include 'labels.j2' %}
  strategy:
    type: Recreate
```
...renders to broken YAML:
```
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    tla: webtool
env: qa
  name: webtool
spec:
  selector:
    matchLabels:
      tla: webtool
env: qa
  strategy:
    type: Recreate
```

deployment_new_syntax.yaml.j2:
```
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    {%* include 'labels.j2' %}
  name: webtool
spec:
  selector:
    matchLabels:
      {%* include 'labels.j2' %}
  strategy:
    type: Recreate
```
...renders correctly:
```
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    tla: webtool
    env: qa
  name: webtool
spec:
  selector:
    matchLabels:
      tla: webtool
      env: qa
  strategy:
    type: Recreate
```

jinja2/filters.py

@@ -580,6 +580,26 @@ def do_indent(
     return rv
 
 
+def do_lineprefix(s, prefix):
+    """Return a copy of the string with each line prepended by the given string.
+
+    :param prefix: String to prepend each line with
+
+    .. versionadded:: 2.11
+        Add block auto-indent feature
+    """
+    newline = u'\n'
+
+    if isinstance(s, Markup):
+        prefix = Markup(prefix)
+        newline = Markup(newline)
+
+    lines = s.splitlines()
+    rv = newline.join(prefix + line if line else line for line in lines)
+
+    return rv
+
+
 @environmentfilter
 def do_truncate(env, s, length=255, killwords=False, end='...', leeway=None):
     """Return a truncated copy of the string. The length is specified
@@ -1158,6 +1178,7 @@ def select_or_reject(args, kwargs, modfunc, lookup_attr):
     'format':               do_format,
     'groupby':              do_groupby,
     'indent':               do_indent,
+    'lineprefix':           do_lineprefix,
     'int':                  do_int,
     'join':                 do_join,
     'last':                 do_last,

jinja2/lexer.py

@@ -487,13 +487,14 @@ def __init__(self, environment):
             'root': [
                 # directives
                 (c('(.*?)(?:%s)' % '|'.join(
-                    [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % (
+                    [r'(?P<raw_begin>(?:\s*%s\-|[ \t]*%s\*|%s)\s*raw\s*(?:\-%s\s*|%s))' % (
+                        e(environment.block_start_string),
                         e(environment.block_start_string),
                         block_prefix_re,
                         e(environment.block_end_string),
                         e(environment.block_end_string)
                     )] + [
-                        r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, prefix_re.get(n,r))
+                        r'(?P<%s_begin>\s*%s\-|[ \t]*%s\*|%s)' % (n, r, r, prefix_re.get(n,r))
                         for n, r in root_tag_rules
                     ])), (TOKEN_DATA, '#bygroup'), '#bygroup'),
                 # data

jinja2/parser.py

@@ -118,23 +118,33 @@ def free_identifier(self, lineno=None):
         nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno)
         return rv
 
-    def parse_statement(self):
+    def parse_statement(self, line_prefix=None):
         """Parse a single statement."""
         token = self.stream.current
         if token.type != 'name':
             self.fail('tag name expected', token.lineno)
         self._tag_stack.append(token.value)
         pop_tag = True
         try:
+            sub = None
             if token.value in _statement_keywords:
-                return getattr(self, 'parse_' + self.stream.current.value)()
+                sub = getattr(self, 'parse_' + self.stream.current.value)()
             if token.value == 'call':
-                return self.parse_call_block()
+                sub = self.parse_call_block()
             if token.value == 'filter':
-                return self.parse_filter_block()
+                sub = self.parse_filter_block()
             ext = self.extensions.get(token.value)
             if ext is not None:
-                return ext(self)
+                sub = ext(self)
+
+            # If auto-indent feature is on '{%* ... %}', wrap statement in a filter
+            if line_prefix:
+                node = nodes.FilterBlock(lineno=token.lineno)
+                node.filter = nodes.Filter(None, 'lineprefix', [nodes.Const(line_prefix)], [], None, None, lineno=token.lineno)
+                node.body = [sub]
+                return node
+            elif sub:
+                return sub
 
             # did not work out, remove the token we pushed by accident
             # from the stack so that the unknown tag fail function can
@@ -877,12 +887,16 @@ def flush_data():
                     add_data(self.parse_tuple(with_condexpr=True))
                     self.stream.expect('variable_end')
                 elif token.type == 'block_begin':
+                    # auto-indented block using {%* ... %}
+                    line_prefix = None
+                    if token.value.endswith('*'):
+                        line_prefix = token.value[:-3]
                     flush_data()
                     next(self.stream)
                     if end_tokens is not None and \
                        self.stream.current.test_any(*end_tokens):
                         return body
-                    rv = self.parse_statement()
+                    rv = self.parse_statement(line_prefix)
                     if isinstance(rv, list):
                         body.extend(rv)
                     else: