From 446546b3a661f21daf489bd243395ac0b658da20 Mon Sep 17 00:00:00 2001 From: Austin Papp Date: Mon, 24 Sep 2018 11:14:59 -0400 Subject: [PATCH 1/7] Adding new script engine --- salt/engines/script.py | 140 ++++++++++++++++++++++++++++++ tests/unit/engines/test_script.py | 64 ++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 salt/engines/script.py create mode 100644 tests/unit/engines/test_script.py diff --git a/salt/engines/script.py b/salt/engines/script.py new file mode 100644 index 000000000000..69cc3da1213e --- /dev/null +++ b/salt/engines/script.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +''' +Send events based on a script's stdout + +Example Config + +.. code-block:: yaml + + engines: + - script: + cmd: /some/script.py -a 1 -b 2 + output: json + interval: 5 + +Script engine configs: + + cmd: Script or command to execute + output: Any available saltstack deserializer + interval: How often in seconds to execute the command + +''' + +from __future__ import absolute_import, print_function +import logging +import shlex +import time +import subprocess + +# import salt libs +import salt.utils.event +import salt.utils.process +import salt.loader +from salt.exceptions import CommandExecutionError + +from salt.ext import six + + +log = logging.getLogger(__name__) + + +def _read_stdout(proc): + ''' + Generator that returns stdout + ''' + for line in iter(proc.stdout.readline, ""): + yield line + + +def _get_serializer(output): + ''' + Helper to return known serializer based on + pass output argument + ''' + serializers = salt.loader.serializers(__opts__) + try: + serializer = getattr(serializers, output) + except AttributeError: + raise CommandExecutionError("Unknown serializer '%s' found for output option", output) + + return serializer + + +def start(cmd, output='json', interval=1): + ''' + Parse stdout of a command and generate an event + + The script engine will scrap stdout of the + given script and generate an event based on the + presence of the 'tag' key and it's value. + + If there is a data obj available, that will also + be fired along with the tag. + + Example: + + Given the following json output from a script: + + { "tag" : "lots/of/tacos", + "data" : { "toppings" : "cilantro" } + } + + This will fire the event 'lots/of/tacos' + on the event bus with the data obj as is. + + :param cmd: The command to execute + :param output: How to deserialize stdout of the script + :param interval: How often to execute the script. + ''' + + try: + cmd = shlex.split(cmd) + except AttributeError: + cmd = shlex.split(six.text_type(cmd)) + log.debug("script engine using command %s", cmd) + + serializer = _get_serializer(output) + + if __opts__.get('__role') == 'master': + fire_master = salt.utils.event.get_master_event( + __opts__, + __opts__['sock_dir']).fire_event + else: + fire_master = __salt__['event.send'] + + while True: + + try: + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + log.debug("Starting script with pid %d", proc.pid) + + for raw_event in _read_stdout(proc): + log.debug(raw_event) + + event = serializer.deserialize(raw_event) + tag = event.get('tag', None) + data = event.get('data', {}) + + if data and 'id' not in data: + data['id'] = __opts__['id'] + + if tag: + log.info("script engine firing event with tag %s", tag) + fire_master(tag=tag, data=data) + + log.debug("Closing script with pid %d", proc.pid) + proc.stdout.close() + rc = proc.wait() + if rc: + raise subprocess.CalledProcessError(rc, cmd) + + except subprocess.CalledProcessError as e: + log.error(e) + finally: + if proc.poll is None: + proc.terminate() + + time.sleep(interval) diff --git a/tests/unit/engines/test_script.py b/tests/unit/engines/test_script.py new file mode 100644 index 000000000000..9abef7f3bff7 --- /dev/null +++ b/tests/unit/engines/test_script.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +''' +unit tests for the script engine +''' +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals +import subprocess +import tempfile + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.paths import TMP +from tests.support.unit import skipIf, TestCase +from tests.support.mock import ( + PropertyMock, + NO_MOCK, + NO_MOCK_REASON, + MagicMock, + mock_open, + patch) + +# Import Salt Libs +import salt.engines.script as script +import salt.config +import salt.utils.stringutils +from salt.exceptions import CommandExecutionError + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class EngineScriptTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test cases for salt.engine.script + ''' + + + def setup_loader_modules(self): + return { + script: { + '__opts__': { + '__role': '', + 'extension_modules' : '', + } + } + } + + + def test__get_serializer(self): + ''' + Test known serializer is returned or exception is raised + if unknown serializer + ''' + self.assertTrue( script._get_serializer('yaml') ) + + with self.assertRaises(CommandExecutionError): + script._get_serializer('bad') + + + def test__read_stdout(self): + ''' + Test we can yield stdout + ''' + with patch('subprocess.Popen') as popen_mock: + popen_mock.stdout.readline.return_value = 'test' + self.assertEqual(next(script._read_stdout(popen_mock)), 'test') + From 22117425504d7531eca3abd9b4d19be3ca628cdd Mon Sep 17 00:00:00 2001 From: Austin Papp Date: Thu, 11 Oct 2018 13:37:17 -0400 Subject: [PATCH 2/7] fixed lint issues --- salt/engines/script.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/salt/engines/script.py b/salt/engines/script.py index 69cc3da1213e..31d88c03234a 100644 --- a/salt/engines/script.py +++ b/salt/engines/script.py @@ -63,15 +63,15 @@ def _get_serializer(output): def start(cmd, output='json', interval=1): ''' Parse stdout of a command and generate an event - - The script engine will scrap stdout of the + + The script engine will scrap stdout of the given script and generate an event based on the presence of the 'tag' key and it's value. - + If there is a data obj available, that will also - be fired along with the tag. - - Example: + be fired along with the tag. + + Example: Given the following json output from a script: @@ -81,7 +81,7 @@ def start(cmd, output='json', interval=1): This will fire the event 'lots/of/tacos' on the event bus with the data obj as is. - + :param cmd: The command to execute :param output: How to deserialize stdout of the script :param interval: How often to execute the script. @@ -108,13 +108,13 @@ def start(cmd, output='json', interval=1): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - log.debug("Starting script with pid %d", proc.pid) + + log.debug("Starting script with pid %d", proc.pid) for raw_event in _read_stdout(proc): log.debug(raw_event) - event = serializer.deserialize(raw_event) + event = serializer.deserialize(raw_event) tag = event.get('tag', None) data = event.get('data', {}) @@ -129,7 +129,7 @@ def start(cmd, output='json', interval=1): proc.stdout.close() rc = proc.wait() if rc: - raise subprocess.CalledProcessError(rc, cmd) + raise subprocess.CalledProcessError(rc, cmd) except subprocess.CalledProcessError as e: log.error(e) From dafe7affdffb57d792a7c6fd28f91fcf09ec6111 Mon Sep 17 00:00:00 2001 From: Austin Papp Date: Thu, 11 Oct 2018 13:45:27 -0400 Subject: [PATCH 3/7] fixed more lint issues --- tests/unit/engines/test_script.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/unit/engines/test_script.py b/tests/unit/engines/test_script.py index 9abef7f3bff7..1cb9e04c1225 100644 --- a/tests/unit/engines/test_script.py +++ b/tests/unit/engines/test_script.py @@ -4,25 +4,19 @@ ''' # Import Python libs from __future__ import absolute_import, print_function, unicode_literals -import subprocess -import tempfile # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin -from tests.support.paths import TMP from tests.support.unit import skipIf, TestCase from tests.support.mock import ( PropertyMock, NO_MOCK, NO_MOCK_REASON, - MagicMock, mock_open, patch) # Import Salt Libs import salt.engines.script as script -import salt.config -import salt.utils.stringutils from salt.exceptions import CommandExecutionError @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -31,13 +25,12 @@ class EngineScriptTestCase(TestCase, LoaderModuleMockMixin): Test cases for salt.engine.script ''' - def setup_loader_modules(self): return { script: { '__opts__': { '__role': '', - 'extension_modules' : '', + 'extension_modules' : '' } } } @@ -48,7 +41,7 @@ def test__get_serializer(self): Test known serializer is returned or exception is raised if unknown serializer ''' - self.assertTrue( script._get_serializer('yaml') ) + self.assertTrue(script._get_serializer('yaml')) with self.assertRaises(CommandExecutionError): script._get_serializer('bad') @@ -61,4 +54,3 @@ def test__read_stdout(self): with patch('subprocess.Popen') as popen_mock: popen_mock.stdout.readline.return_value = 'test' self.assertEqual(next(script._read_stdout(popen_mock)), 'test') - From ea2c9cdadbacc141b640a218f4a30b0222e92321 Mon Sep 17 00:00:00 2001 From: Austin Papp Date: Thu, 11 Oct 2018 14:03:37 -0400 Subject: [PATCH 4/7] more linting --- tests/unit/engines/test_script.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/unit/engines/test_script.py b/tests/unit/engines/test_script.py index 1cb9e04c1225..a3ee832aa1d1 100644 --- a/tests/unit/engines/test_script.py +++ b/tests/unit/engines/test_script.py @@ -9,16 +9,15 @@ from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import skipIf, TestCase from tests.support.mock import ( - PropertyMock, NO_MOCK, NO_MOCK_REASON, - mock_open, patch) # Import Salt Libs import salt.engines.script as script from salt.exceptions import CommandExecutionError + @skipIf(NO_MOCK, NO_MOCK_REASON) class EngineScriptTestCase(TestCase, LoaderModuleMockMixin): ''' @@ -30,12 +29,11 @@ def setup_loader_modules(self): script: { '__opts__': { '__role': '', - 'extension_modules' : '' + 'extension_modules': '' } } } - def test__get_serializer(self): ''' Test known serializer is returned or exception is raised @@ -46,7 +44,6 @@ def test__get_serializer(self): with self.assertRaises(CommandExecutionError): script._get_serializer('bad') - def test__read_stdout(self): ''' Test we can yield stdout From 957a7d3c1ebbdccde9d53ee1aa31b8ae899274d1 Mon Sep 17 00:00:00 2001 From: Austin Papp Date: Tue, 16 Oct 2018 16:43:14 -0400 Subject: [PATCH 5/7] minor fix --- salt/engines/script.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/salt/engines/script.py b/salt/engines/script.py index 31d88c03234a..307099be3ec6 100644 --- a/salt/engines/script.py +++ b/salt/engines/script.py @@ -53,12 +53,10 @@ def _get_serializer(output): ''' serializers = salt.loader.serializers(__opts__) try: - serializer = getattr(serializers, output) + return getattr(serializers, output) except AttributeError: raise CommandExecutionError("Unknown serializer '%s' found for output option", output) - return serializer - def start(cmd, output='json', interval=1): ''' @@ -86,7 +84,6 @@ def start(cmd, output='json', interval=1): :param output: How to deserialize stdout of the script :param interval: How often to execute the script. ''' - try: cmd = shlex.split(cmd) except AttributeError: From cbb179a93fb75624d8ec76089c4663126f586295 Mon Sep 17 00:00:00 2001 From: Austin Papp Date: Tue, 16 Oct 2018 16:45:05 -0400 Subject: [PATCH 6/7] fully populated opts dunder --- tests/unit/engines/test_script.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/unit/engines/test_script.py b/tests/unit/engines/test_script.py index a3ee832aa1d1..998a3dd4b935 100644 --- a/tests/unit/engines/test_script.py +++ b/tests/unit/engines/test_script.py @@ -14,6 +14,7 @@ patch) # Import Salt Libs +import salt.config import salt.engines.script as script from salt.exceptions import CommandExecutionError @@ -25,12 +26,11 @@ class EngineScriptTestCase(TestCase, LoaderModuleMockMixin): ''' def setup_loader_modules(self): + + opts = salt.config.DEFAULT_MASTER_OPTS return { - script: { - '__opts__': { - '__role': '', - 'extension_modules': '' - } + script: { + '__opts__': opts } } @@ -39,7 +39,8 @@ def test__get_serializer(self): Test known serializer is returned or exception is raised if unknown serializer ''' - self.assertTrue(script._get_serializer('yaml')) + for serializers in ('json', 'yaml', 'msgpack'): + self.assertTrue(script._get_serializer(serializers)) with self.assertRaises(CommandExecutionError): script._get_serializer('bad') From 6d616c56467dec38e41d81fc66656b1cfbf68911 Mon Sep 17 00:00:00 2001 From: Austin Papp Date: Tue, 16 Oct 2018 17:15:54 -0400 Subject: [PATCH 7/7] whitespace --- tests/unit/engines/test_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/engines/test_script.py b/tests/unit/engines/test_script.py index 998a3dd4b935..8ddd712043cb 100644 --- a/tests/unit/engines/test_script.py +++ b/tests/unit/engines/test_script.py @@ -29,7 +29,7 @@ def setup_loader_modules(self): opts = salt.config.DEFAULT_MASTER_OPTS return { - script: { + script: { '__opts__': opts } }