Merge pull request #51718 from arizvisa/GH-51711

Fix the Cheetah template renderer in salt.utils.templates, and unit-tests for Jinja, Cheetah, Mako, Genshi, wempy templates,

requirements/static/darwin.in

@@ -27,3 +27,9 @@ supervisor==3.3.5; python_version < '3'
 virtualenv
 watchdog
 yamlordereddictloader
+
+# Available template libraries that can be used
+Genshi
+Cheetah3==3.1.0
+Mako
+wempy; python_version <'3'

requirements/static/linux.in

@@ -41,3 +41,9 @@ timelib
 tornado<5.0
 virtualenv
 watchdog
+
+# Available template libraries that can be used
+Genshi
+Cheetah3==3.1.0
+Mako
+wempy; python_version <'3'

requirements/static/py2.7/darwin.txt

@@ -134,3 +134,5 @@ zc.lockfile==1.4          # via cherrypy
 zipp==0.6.0               # via importlib-metadata
 # Passthrough dependencies from pkg/osx/req.txt
 pyobjc==5.1.2
+# Passthrough dependencies from pkg/osx/req.txt
+pyobjc==5.1.2

requirements/static/py3.4/linux.txt

@@ -10,9 +10,8 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.3.0       # via pytest
 attrs==19.1.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-backports-abc==0.5        # via tornado
 backports.functools-lru-cache==1.5  # via cheroot
-backports.ssl-match-hostname==3.7.0.1  # via docker, websocket-client
+backports.ssl-match-hostname==3.7.0.1  # via docker
 bcrypt==3.1.6             # via paramiko
 boto3==1.9.132
 boto==2.49.0
@@ -21,6 +20,7 @@ cachetools==3.1.0         # via google-auth
 certifi==2019.3.9
 cffi==1.12.2
 chardet==3.0.4            # via requests
+cheetah3==3.1.0
 cheroot==6.5.4            # via cherrypy
 cherrypy==17.3.0
 contextlib2==0.5.5        # via cherrypy
@@ -32,6 +32,7 @@ docker==3.7.2
 docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 future==0.17.1            # via python-jose
+genshi==0.7.3
 gitdb2==2.0.5             # via gitpython
 gitpython==2.1.11
 google-auth==1.6.3        # via kubernetes
@@ -51,6 +52,7 @@ kazoo==2.6.1
 keyring==5.7.1
 kubernetes==3.0.0
 lxml==4.3.3               # via junos-eznc, ncclient
+mako==1.1.0
 markupsafe==1.1.1
 mock==3.0.5 ; python_version < "3.6"
 more-itertools==5.0.0
@@ -101,8 +103,7 @@ scp==0.13.2               # via junos-eznc
 selectors2==2.0.1         # via ncclient
 setproctitle==1.1.10
 setuptools-scm==3.2.0
-singledispatch==3.4.0.3   # via tornado
-six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client
+six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client
 smmap2==2.0.5             # via gitdb2
 strict-rfc3339==0.7
 tempora==1.14.1           # via portend

requirements/static/py3.5/darwin.txt

@@ -10,9 +10,9 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.3.0       # via pytest
 attrs==19.1.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-backports-abc==0.5
 backports.functools-lru-cache==1.5  # via cheroot
 backports.ssl_match_hostname==3.7.0.1
+backports_abc==0.5
 bcrypt==3.1.6             # via paramiko
 boto3==1.9.132
 boto==2.49.0
@@ -21,6 +21,7 @@ cachetools==3.1.0         # via google-auth
 certifi==2019.3.9
 cffi==1.12.2
 chardet==3.0.4            # via requests
+cheetah3==3.1.0
 cheroot==6.5.5            # via cherrypy
 cherrypy==17.4.1
 click==7.0
@@ -35,6 +36,7 @@ docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 enum34==1.1.6
 future==0.17.1            # via python-jose
+genshi==0.7.3
 gitdb2==2.0.5             # via gitpython
 gitdb==0.6.4
 gitpython==2.1.11

requirements/static/py3.5/linux.txt

@@ -10,9 +10,7 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.3.0       # via pytest
 attrs==19.1.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-backports-abc==0.5        # via tornado
 backports.functools-lru-cache==1.5  # via cheroot
-backports.ssl-match-hostname==3.7.0.1  # via websocket-client
 bcrypt==3.1.6             # via paramiko
 boto3==1.9.132
 boto==2.49.0
@@ -21,6 +19,7 @@ cachetools==3.1.0         # via google-auth
 certifi==2019.3.9
 cffi==1.12.2
 chardet==3.0.4            # via requests
+cheetah3==3.1.0
 cheroot==6.5.4            # via cherrypy
 cherrypy==17.3.0
 contextlib2==0.5.5        # via cherrypy
@@ -32,6 +31,7 @@ docker==3.7.2
 docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 future==0.17.1            # via python-jose
+genshi==0.7.3
 gitdb2==2.0.5             # via gitpython
 gitpython==2.1.11
 google-auth==1.6.3        # via kubernetes
@@ -51,6 +51,7 @@ kazoo==2.6.1
 keyring==5.7.1
 kubernetes==3.0.0
 lxml==4.3.3               # via junos-eznc, ncclient
+mako==1.1.0
 markupsafe==1.1.1
 mock==3.0.5 ; python_version < "3.6"
 more-itertools==5.0.0
@@ -99,8 +100,7 @@ salttesting==2017.6.1
 scp==0.13.2               # via junos-eznc
 setproctitle==1.1.10
 setuptools-scm==3.2.0
-singledispatch==3.4.0.3   # via tornado
-six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client
+six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pathlib2, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client
 smmap2==2.0.5             # via gitdb2
 strict-rfc3339==0.7
 tempora==1.14.1           # via portend

requirements/static/py3.5/windows.txt

@@ -19,6 +19,7 @@ cachetools==3.1.0         # via google-auth
 certifi==2019.3.9
 cffi==1.12.2
 chardet==3.0.4            # via requests
+cheetah3==3.1.0
 cheroot==6.5.5            # via cherrypy
 cherrypy==17.4.1
 colorama==0.4.1           # via pytest
@@ -32,6 +33,7 @@ docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 enum34==1.1.6
 future==0.17.1            # via python-jose
+genshi==0.7.3
 gitdb2==2.0.5             # via gitpython
 gitdb==0.6.4
 gitpython==2.1.10

requirements/static/py3.6/darwin.txt

@@ -10,9 +10,9 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.3.0       # via pytest
 attrs==19.1.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-backports-abc==0.5
 backports.functools-lru-cache==1.5  # via cheroot
 backports.ssl_match_hostname==3.7.0.1
+backports_abc==0.5
 bcrypt==3.1.6             # via paramiko
 boto3==1.9.132
 boto==2.49.0
@@ -21,6 +21,7 @@ cachetools==3.1.0         # via google-auth
 certifi==2019.3.9
 cffi==1.12.2
 chardet==3.0.4            # via requests
+cheetah3==3.1.0
 cheroot==6.5.5            # via cherrypy
 cherrypy==17.4.1
 click==7.0
@@ -35,6 +36,7 @@ docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 enum34==1.1.6
 future==0.17.1            # via python-jose
+genshi==0.7.3
 gitdb2==2.0.5             # via gitpython
 gitdb==0.6.4
 gitpython==2.1.11

requirements/static/py3.6/linux.txt

@@ -10,9 +10,7 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.3.0       # via pytest
 attrs==19.1.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-backports-abc==0.5        # via tornado
 backports.functools-lru-cache==1.5  # via cheroot
-backports.ssl-match-hostname==3.7.0.1  # via websocket-client
 bcrypt==3.1.6             # via paramiko
 boto3==1.9.132
 boto==2.49.0
@@ -21,6 +19,7 @@ cachetools==3.1.0         # via google-auth
 certifi==2019.3.9
 cffi==1.12.2
 chardet==3.0.4            # via requests
+cheetah3==3.1.0
 cheroot==6.5.4            # via cherrypy
 cherrypy==17.3.0
 contextlib2==0.5.5        # via cherrypy
@@ -32,6 +31,7 @@ docker==3.7.2
 docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 future==0.17.1            # via python-jose
+genshi==0.7.3
 gitdb2==2.0.5             # via gitpython
 gitpython==2.1.11
 google-auth==1.6.3        # via kubernetes
@@ -51,6 +51,7 @@ kazoo==2.6.1
 keyring==5.7.1
 kubernetes==3.0.0
 lxml==4.3.3               # via junos-eznc, ncclient
+mako==1.1.0
 markupsafe==1.1.1
 mock==2.0.0               # via moto
 more-itertools==5.0.0
@@ -99,8 +100,7 @@ salttesting==2017.6.1
 scp==0.13.2               # via junos-eznc
 setproctitle==1.1.10
 setuptools-scm==3.2.0
-singledispatch==3.4.0.3   # via tornado
-six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client
+six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client
 smmap2==2.0.5             # via gitdb2
 strict-rfc3339==0.7
 tempora==1.14.1           # via portend

requirements/static/py3.6/windows.txt

@@ -19,6 +19,7 @@ cachetools==3.1.0         # via google-auth
 certifi==2019.3.9
 cffi==1.12.2
 chardet==3.0.4            # via requests
+cheetah3==3.1.0
 cheroot==6.5.5            # via cherrypy
 cherrypy==17.4.1
 colorama==0.4.1           # via pytest
@@ -32,6 +33,7 @@ docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 enum34==1.1.6
 future==0.17.1            # via python-jose
+genshi==0.7.3
 gitdb2==2.0.5             # via gitpython
 gitdb==0.6.4
 gitpython==2.1.10

requirements/static/py3.7/darwin.txt

@@ -10,9 +10,9 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.3.0       # via pytest
 attrs==19.1.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-backports-abc==0.5
 backports.functools-lru-cache==1.5  # via cheroot
 backports.ssl_match_hostname==3.7.0.1
+backports_abc==0.5
 bcrypt==3.1.6             # via paramiko
 boto3==1.9.132
 boto==2.49.0
@@ -21,6 +21,7 @@ cachetools==3.1.0         # via google-auth
 certifi==2019.3.9
 cffi==1.12.2
 chardet==3.0.4            # via requests
+cheetah3==3.1.0
 cheroot==6.5.5            # via cherrypy
 cherrypy==17.4.1
 click==7.0
@@ -35,6 +36,7 @@ docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 enum34==1.1.6
 future==0.17.1            # via python-jose
+genshi==0.7.3
 gitdb2==2.0.5             # via gitpython
 gitdb==0.6.4
 gitpython==2.1.11

requirements/static/py3.7/linux.txt

@@ -10,9 +10,7 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.3.0       # via pytest
 attrs==19.1.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-backports-abc==0.5        # via tornado
 backports.functools-lru-cache==1.5  # via cheroot
-backports.ssl-match-hostname==3.7.0.1  # via websocket-client
 bcrypt==3.1.6             # via paramiko
 boto3==1.9.132
 boto==2.49.0
@@ -21,6 +19,7 @@ cachetools==3.1.0         # via google-auth
 certifi==2019.3.9
 cffi==1.12.2
 chardet==3.0.4            # via requests
+cheetah3==3.1.0
 cheroot==6.5.4            # via cherrypy
 cherrypy==17.3.0
 contextlib2==0.5.5        # via cherrypy
@@ -32,6 +31,7 @@ docker==3.7.2
 docutils==0.14            # via botocore
 ecdsa==0.13.3             # via python-jose
 future==0.17.1            # via python-jose
+genshi==0.7.3
 gitdb2==2.0.5             # via gitpython
 gitpython==2.1.11
 google-auth==1.6.3        # via kubernetes
@@ -51,6 +51,7 @@ kazoo==2.6.1
 keyring==5.7.1
 kubernetes==3.0.0
 lxml==4.3.3               # via junos-eznc, ncclient
+mako==1.1.0
 markupsafe==1.1.1
 mock==2.0.0               # via moto
 more-itertools==5.0.0
@@ -99,8 +100,7 @@ salttesting==2017.6.1
 scp==0.13.2               # via junos-eznc
 setproctitle==1.1.10
 setuptools-scm==3.2.0
-singledispatch==3.4.0.3   # via tornado
-six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, singledispatch, tempora, websocket-client
+six==1.12.0               # via bcrypt, cheroot, cherrypy, cryptography, docker, docker-pycreds, google-auth, junos-eznc, kazoo, kubernetes, mock, more-itertools, moto, ncclient, packaging, pygit2, pynacl, pyopenssl, pytest, python-dateutil, python-jose, pyvmomi, responses, salttesting, tempora, websocket-client
 smmap2==2.0.5             # via gitdb2
 strict-rfc3339==0.7
 tempora==1.14.1           # via portend

requirements/static/windows.in

@@ -31,3 +31,9 @@ virtualenv
 timelib
 tornado<5.0
 wmi==1.4.9
+
+# Available template libraries that can be used
+Genshi
+Cheetah3==3.1.0
+Mako
+wempy; python_version <'3'

salt/utils/templates.py

@@ -520,7 +520,26 @@ def render_cheetah_tmpl(tmplstr, context, tmplpath=None):
     Render a Cheetah template.
     '''
     from Cheetah.Template import Template
-    return salt.utils.data.decode(Template(tmplstr, searchList=[context]))
+
+    # Compile the template and render it into the class
+    tclass = Template.compile(tmplstr)
+    data = tclass(namespaces=[context])
+
+    # Figure out which method to call based on the type of tmplstr
+    if six.PY3 and isinstance(tmplstr, six.string_types):
+        # This should call .__unicode__()
+        res = str(data)
+    elif six.PY2 and isinstance(tmplstr, six.text_type):
+        # Expicitly call .__unicode__()
+        res = data.__unicode__()
+    elif isinstance(tmplstr, six.binary_type):
+        # This should call .__str()
+        res = str(data)
+    else:
+        raise SaltRenderError('Unknown type {!s} for Cheetah template while trying to render.'.format(type(tmplstr)))
+
+    # Now we can decode it to the correct encoding
+    return salt.utils.data.decode(res)
 # pylint: enable=3rd-party-module-not-gated