[master] Port #51047 (Jinja map troubleshooting)

doc/ref/modules/all/index.rst

@@ -199,6 +199,7 @@ execution modules
     jboss7
     jboss7_cli
     jenkinsmod
+    jinja
     jira_mod
     junos
     k8s

doc/ref/modules/all/salt.modules.jinja.rst

@@ -0,0 +1,6 @@
+==================
+salt.modules.jinja
+==================
+
+.. automodule:: salt.modules.jinja
+    :members:

doc/topics/releases/neon.rst

@@ -52,6 +52,34 @@ as well as managing keystore files.
             Hn+GmxZA
             -----END CERTIFICATE-----
 
+
+Troubleshooting Jinja map files
+===============================
+
+A new :py:func:`execution module <salt.modules.jinja>` for ``map.jinja`` troubleshooting
+has been added.
+
+Assuming the map is loaded in your formula SLS as follows:
+
+.. code-block:: jinja
+
+  {% from "myformula/map.jinja" import myformula with context %}
+
+The following command can be used to load the map and check the results:
+
+.. code-block:: bash
+
+  salt myminion jinja.load_map myformula/map.jinja myformula
+
+The module can be also used to test ``json`` and ``yaml`` maps:
+
+.. code-block:: bash
+
+  salt myminion jinja.import_yaml myformula/defaults.yaml
+
+  salt myminion jinja.import_json myformula/defaults.json
+
+
 Slot Syntax Updates
 ===================
 

salt/modules/jinja.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+'''
+Module for checking jinja maps and verifying the result of loading JSON/YAML
+files
+
+.. versionadded:: Neon
+'''
+from __future__ import absolute_import, print_function, unicode_literals
+
+# Import Python libs
+import functools
+import logging
+import textwrap
+
+# Import Salt libs
+import salt.loader
+import salt.template
+import salt.utils.json
+
+log = logging.getLogger(__name__)
+
+
+def _strip_odict(wrapped):
+    '''
+    dump to json and load it again, replaces OrderedDicts with regular ones
+    '''
+    @functools.wraps(wrapped)
+    def strip(*args):
+        return salt.utils.json.loads(salt.utils.json.dumps(wrapped(*args)))
+    return strip
+
+
+@_strip_odict
+def load_map(path, value):
+    '''
+    Loads the map at the specified path, and returns the specified value from
+    that map.
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        # Assuming the map is loaded in your formula SLS as follows:
+        #
+        # {% from "myformula/map.jinja" import myformula with context %}
+        #
+        # the following syntax can be used to load the map and check the
+        # results:
+        salt myminion jinja.load_map myformula/map.jinja myformula
+    '''
+    tmplstr = textwrap.dedent('''\
+        {{% from "{path}" import {value} with context %}}
+        {{{{ {value} | tojson }}}}
+        '''.format(path=path, value=value))
+    return salt.template.compile_template_str(
+        tmplstr,
+        salt.loader.render(__opts__, __salt__),
+        __opts__['renderer'],
+        __opts__['renderer_blacklist'],
+        __opts__['renderer_whitelist'])
+
+
+@_strip_odict
+def import_yaml(path):
+    '''
+    Loads YAML data from the specified path
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt myminion jinja.import_yaml myformula/foo.yaml
+    '''
+    tmplstr = textwrap.dedent('''\
+        {{% import_yaml "{path}" as imported %}}
+        {{{{ imported | tojson }}}}
+        '''.format(path=path))
+    return salt.template.compile_template_str(
+        tmplstr,
+        salt.loader.render(__opts__, __salt__),
+        __opts__['renderer'],
+        __opts__['renderer_blacklist'],
+        __opts__['renderer_whitelist'])
+
+
+@_strip_odict
+def import_json(path):
+    '''
+    Loads JSON data from the specified path
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt myminion jinja.import_JSON myformula/foo.json
+    '''
+    tmplstr = textwrap.dedent('''\
+        {{% import_json "{path}" as imported %}}
+        {{{{ imported | tojson }}}}
+        '''.format(path=path))
+    return salt.template.compile_template_str(
+        tmplstr,
+        salt.loader.render(__opts__, __salt__),
+        __opts__['renderer'],
+        __opts__['renderer_blacklist'],
+        __opts__['renderer_whitelist'])

tests/integration/files/file/base/modules/jinja/defaults.yaml

@@ -0,0 +1,13 @@
+---
+template:
+  pkg:
+    name: template
+  rootgroup: root
+  config: '/etc/template'
+  service:
+    name: template
+  subcomponent:
+    config: '/etc/template-subcomponent-formula.conf'
+  # Just here for testing
+  added_in_defaults: defaults_value
+  winner: defaults

tests/integration/files/file/base/modules/jinja/map.jinja

@@ -0,0 +1,49 @@
+{%- set tplroot = tpldir.split('/')[0] %}
+
+{%- import_yaml tplroot ~ "/defaults.yaml" as default_settings %}
+{%- import_json tplroot ~ "/osarchmap.json" as osarchmap %}
+{%- import_yaml tplroot ~ "/osfamilymap.yaml" as osfamilymap %}
+{%- import_yaml tplroot ~ "/osmap.yaml" as osmap %}
+{%- import_yaml tplroot ~ "/osfingermap.yaml" as osfingermap %}
+
+{%- set _config = salt['config.get'](tplroot, default={}) %}
+
+{%- set defaults = salt['grains.filter_by'](
+      default_settings,
+      default='template',
+      merge=salt['grains.filter_by'](
+        osarchmap,
+        grain='osarch',
+        merge=salt['grains.filter_by'](
+          osfamilymap,
+          grain='os_family',
+          merge=salt['grains.filter_by'](
+            osmap,
+            grain='os',
+            merge=salt['grains.filter_by'](
+              osfingermap,
+              grain='osfinger',
+              merge=salt['grains.filter_by'](
+                _config,
+                default='lookup'
+              )
+            )
+          )
+        )
+      )
+    )
+%}
+
+{%- set config = salt['grains.filter_by'](
+      {'defaults': defaults},
+      default='defaults',
+      merge=_config
+    )
+%}
+
+{%- set template = config %}
+
+{%- if grains.os == 'MacOS' %}
+{%-   set macos_group = salt['cmd.run']("stat -f '%Sg' /dev/console") %}
+{%-   do template.update({'rootgroup': macos_group}) %}
+{%- endif %}

tests/integration/files/file/base/modules/jinja/osarchmap.json

@@ -0,0 +1,26 @@
+{
+  "386": {
+    "arch": 386
+  },
+  "amd64": {
+    "arch": "amd64"
+  },
+  "x86_64": {
+    "arch": "amd64"
+  },
+  "arm64": {
+    "arch": "arm64"
+  },
+  "armv6l": {
+    "arch": "armv6l"
+  },
+  "armv7l": {
+    "arch": "armv7l"
+  },
+  "ppc64le": {
+    "arch": "ppc64le"
+  },
+  "s390x": {
+    "arch": "s390x"
+  }
+}

tests/integration/files/file/base/modules/jinja/osfamilymap.yaml

@@ -0,0 +1,36 @@
+---
+Debian:
+  pkg:
+    name: template-debian
+  config: /etc/template.d/custom.conf
+
+RedHat:
+  pkg:
+    name: template-redhat
+  config: /etc/template.conf
+
+Suse:
+  pkg:
+    name: template-suse
+
+Gentoo: {}
+
+Arch:
+  pkg:
+    name: template-arch
+  service:
+    name: service-arch
+
+Alpine: {}
+
+FreeBSD:
+  rootgroup: wheel
+
+OpenBSD:
+  rootgroup: wheel
+
+Solaris: {}
+
+Windows: {}
+
+MacOS: {}

tests/integration/files/file/base/modules/jinja/osfingermap.yaml

@@ -0,0 +1,11 @@
+---
+# os: Ubuntu
+Ubuntu-18.04:
+  config: /etc/template.d/custom-ubuntu-18.04.conf
+
+# os: CentOS
+CentOS-6:
+  pkg:
+    name: template-centos-6
+  config: /etc/template.d/custom-centos-6.conf
+CentOS-7: {}

tests/integration/files/file/base/modules/jinja/osmap.yaml

@@ -0,0 +1,29 @@
+---
+# os_family: Debian
+Ubuntu:
+  pkg:
+    name: template-ubuntu
+  config: /etc/template.d/custom-ubuntu.conf
+
+Raspbian: {}
+
+# os_family: RedHat
+Fedora:
+  pkg:
+    name: template-fedora
+  service:
+    name: service-fedora
+
+CentOS: {}
+
+# os_family: Suse
+openSUSE: {}
+
+# os_family: Gentoo
+Funtoo: {}
+
+# os_family: Arch
+Manjaro: {}
+
+# os_family: Solaris
+SmartOS: {}

tests/integration/modules/test_jinja.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+'''
+Test the jinja module
+'''
+
+# Import python libs
+from __future__ import absolute_import, print_function, unicode_literals
+import os
+
+# Import Salt Testing libs
+from tests.support.runtests import RUNTIME_VARS
+from tests.support.case import ModuleCase
+from tests.support.helpers import requires_system_grains
+
+# Import Salt libs
+import salt.utils.json
+import salt.utils.files
+import salt.utils.yaml
+
+
+class TestModulesJinja(ModuleCase):
+    '''
+    Test the jinja map module
+    '''
+
+    def _path(self, name, absolute=False):
+        path = os.path.join('modules', 'jinja', name)
+        if absolute:
+            return os.path.join(RUNTIME_VARS.BASE_FILES, path)
+        else:
+            return path
+
+    def test_import_json(self):
+        json_file = 'osarchmap.json'
+        ret = self.run_function('jinja.import_json', [self._path(json_file)])
+        with salt.utils.files.fopen(self._path(json_file, absolute=True)) as fh_:
+            self.assertDictEqual(salt.utils.json.load(fh_), ret)
+
+    def test_import_yaml(self):
+        yaml_file = 'defaults.yaml'
+        ret = self.run_function('jinja.import_yaml', [self._path(yaml_file)])
+        with salt.utils.files.fopen(self._path(yaml_file, absolute=True)) as fh_:
+            self.assertDictEqual(salt.utils.yaml.safe_load(fh_), ret)
+
+    @requires_system_grains
+    def test_load_map(self, grains):
+        ret = self.run_function('jinja.load_map', [self._path('map.jinja'), 'template'])
+
+        with salt.utils.files.fopen(self._path('defaults.yaml', absolute=True)) as fh_:
+            defaults = salt.utils.yaml.safe_load(fh_)
+        with salt.utils.files.fopen(self._path('osarchmap.json', absolute=True)) as fh_:
+            osarchmap = salt.utils.json.load(fh_)
+        with salt.utils.files.fopen(self._path('osfamilymap.yaml', absolute=True)) as fh_:
+            osfamilymap = salt.utils.yaml.safe_load(fh_)
+        with salt.utils.files.fopen(self._path('osmap.yaml', absolute=True)) as fh_:
+            osmap = salt.utils.yaml.safe_load(fh_)
+        with salt.utils.files.fopen(self._path('osfingermap.yaml', absolute=True)) as fh_:
+            osfingermap = salt.utils.yaml.safe_load(fh_)
+
+        self.assertEqual(ret.get('arch'), osarchmap.get(grains['osarch'], {}).get('arch'))
+        self.assertEqual(
+            ret.get('config'),
+            osfingermap.get(
+                grains['osfinger'], {}
+            ).get('config', osmap.get(
+                grains['os'], {}
+            ).get('config', osfamilymap.get(
+                grains['os_family'], {}
+            ).get('config', defaults.get(
+                'template'
+            ).get('config'))))
+        )