Add Regorus policy engine framework (#3187)

* Add policy engine framework and Regorus executable

* Add comments and address Pylint issues

* Add unit tests

* Fix Pylint issues for unit tests

* Add TODOs

* Add TODOs

* Add check for ARM64

* Create tempfile for set_input

* Address comments

* Fix UT failures

* Remove TODOs

* Raise exceptions

* Address comments

* Fix UT failures

* Address comments

* Pylint fix

* Pylint fix

* Add more UT

* Remove regorus param validation

* Address comments

* Pylint

* Address comments

* Address comments

* Pylint

* Address comments

* Address comments

azurelinuxagent/common/conf.py

@@ -147,7 +147,8 @@ def load_conf_from_file(conf_file_path, conf=__conf__):
     "Debug.EnableAgentMemoryUsageCheck": False,
     "Debug.EnableFastTrack": True,
     "Debug.EnableGAVersioning": True,
-    "Debug.EnableCgroupV2ResourceLimiting": False
+    "Debug.EnableCgroupV2ResourceLimiting": False,
+    "Debug.EnableExtensionPolicy": False
 }
 
 
@@ -683,7 +684,15 @@ def get_firewall_rules_log_period(conf=__conf__):
     """
     return conf.get_int("Debug.FirewallRulesLogPeriod", 86400)
 
+
+def get_extension_policy_enabled(conf=__conf__):
+    """
+    Determine whether extension policy is enabled. If true, policy will be enforced before installing any extensions.
+    NOTE: This option is experimental and may be removed in later versions of the Agent.
+    """
+    return conf.get_switch("Debug.EnableExtensionPolicy", False)
 
+
 def get_enable_cgroup_v2_resource_limiting(conf=__conf__):
     """
     If True, the agent will enable resource monitoring and enforcement for the log collector on machines using cgroup v2.

azurelinuxagent/common/event.py

@@ -86,6 +86,7 @@ class WALAEventOperation:
     Downgrade = "Downgrade"
     Download = "Download"
     Enable = "Enable"
+    ExtensionPolicy = "ExtensionPolicy"
     ExtensionProcessing = "ExtensionProcessing"
     ExtensionTelemetryEventProcessing = "ExtensionTelemetryEventProcessing"
     FetchGoalState = "FetchGoalState"
@@ -111,6 +112,7 @@ class WALAEventOperation:
     OpenSsl = "OpenSsl"
     Partition = "Partition"
     PersistFirewallRules = "PersistFirewallRules"
+    Policy = "Policy"
     ProvisionAfterExtensions = "ProvisionAfterExtensions"
     PluginSettingsVersionMismatch = "PluginSettingsVersionMismatch"
     InvalidExtensionConfig = "InvalidExtensionConfig"

azurelinuxagent/ga/exthandlers.py

@@ -476,6 +476,7 @@ def handle_ext_handlers(self, goal_state_id):
 
         depends_on_err_msg = None
         extensions_enabled = conf.get_extensions_enabled()
+
         for extension, ext_handler in all_extensions:
 
             handler_i = ExtHandlerInstance(ext_handler, self.protocol, extension=extension)

azurelinuxagent/ga/policy/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Requires Python 2.6+ and Openssl 1.0+

azurelinuxagent/ga/policy/policy_engine.py

@@ -0,0 +1,169 @@
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Requires Python 2.4+ and Openssl 1.0+
+#
+
+from azurelinuxagent.common import logger
+from azurelinuxagent.common.version import DISTRO_VERSION, DISTRO_NAME
+from azurelinuxagent.common.utils.distro_version import DistroVersion
+from azurelinuxagent.common.event import WALAEventOperation, add_event
+from azurelinuxagent.common import conf
+from azurelinuxagent.common.osutil import get_osutil
+import azurelinuxagent.ga.policy.regorus as regorus
+from azurelinuxagent.ga.policy.regorus import PolicyError
+
+# Define support matrix for Regorus and policy engine feature.
+# Dict in the format: { distro:min_supported_version }
+POLICY_SUPPORTED_DISTROS_MIN_VERSIONS = {
+    'ubuntu': DistroVersion('16.04'),
+    'mariner': DistroVersion('2'),
+    'azurelinux': DistroVersion('3')
+}
+# TODO: add 'arm64', 'aarch64' here once support is enabled for ARM64
+POLICY_SUPPORTED_ARCHITECTURE = ['x86_64']
+
+
+class PolicyEngine(object):
+    """
+    Implements base policy engine API.
+    If any errors are thrown in regorus.py, they will be caught and re-raised here.
+    The caller will be responsible for handling errors.
+    """
+    def __init__(self, rule_file, policy_file):
+        """
+        Constructor checks that policy enforcement should be enabled, and then sets up the
+        Regorus policy engine (add rule and policy file).
+
+        rule_file: Path to a Rego file that specifies rules for policy behavior.
+
+        policy_file: Path to a JSON file that specifies parameters for policy behavior - for example,
+        whether allowlist or extension signing should be enforced.
+        The expected file format is:
+        {
+            "azureGuestAgentPolicy": {
+                "policyVersion": "0.1.0",
+                "signingRules": {
+                    "extensionSigned": <true, false>
+                },
+                "allowListOnly": <true, false>
+            },
+            "azureGuestExtensionsPolicy": {
+                "allowed_ext_1": {
+                    "signingRules": {
+                        "extensionSigned": <true, false>
+                    }
+                }
+        }
+        """
+        self._engine = None
+        if not self.is_policy_enforcement_enabled():
+            self._log_policy(msg="Policy enforcement is not enabled.")
+            return
+
+        # If unsupported, this call will raise an error
+        self._check_policy_enforcement_supported()
+        self._engine = regorus.Engine(policy_file=policy_file, rule_file=rule_file)
+
+    @classmethod
+    def _log_policy(cls, msg, is_success=True, op=WALAEventOperation.Policy, send_event=True):
+        """
+        Log information to console and telemetry.
+        """
+        if is_success:
+            logger.info(msg)
+        else:
+            logger.error(msg)
+        if send_event:
+            add_event(op=op, message=msg, is_success=is_success)
+
+    @staticmethod
+    def is_policy_enforcement_enabled():
+        """
+        Check whether user has opted into policy enforcement feature.
+        Caller function should check this before performing any operations.
+        """
+        # TODO: The conf flag will be removed post private preview. Before public preview, add checks
+        # according to the planned user experience (TBD).
+        return conf.get_extension_policy_enabled()
+
+    @staticmethod
+    def _check_policy_enforcement_supported():
+        """
+        Check that both platform architecture and distro/version are supported.
+        If supported, do nothing.
+        If not supported, raise PolicyError with user-friendly error message.
+        """
+        osutil = get_osutil()
+        arch = osutil.get_vm_arch()
+        # TODO: surface as a user error with clear instructions for fixing
+        msg = "Attempted to enable policy enforcement, but feature is not supported on "
+        if arch not in POLICY_SUPPORTED_ARCHITECTURE:
+            msg += " architecture " + str(arch)
+        elif DISTRO_NAME not in POLICY_SUPPORTED_DISTROS_MIN_VERSIONS:
+            msg += " distro " + str(DISTRO_NAME)
+        else:
+            min_version = POLICY_SUPPORTED_DISTROS_MIN_VERSIONS.get(DISTRO_NAME)
+            if DISTRO_VERSION < min_version:
+                msg += " distro " + DISTRO_NAME + " " + DISTRO_VERSION + ". Policy is only supported on version " + \
+                        str(min_version) + " and above."
+            else:
+                return  # do nothing if platform is supported
+        raise PolicyError(msg)
+
+    def evaluate_query(self, input_to_check, query):
+        """
+        Input_to_check is the input we want to check against the policy engine (ex: extensions we want to install).
+        Input_to_check should be a dict. Expected format:
+        {
+            "extensions": {
+                "<extension_name_1>": {
+                    "signingInfo": {
+                        "extensionSigned": <true, false>
+                    }
+                }, ...
+        }
+
+        The query parameter specifies the value we want to retrieve from the policy engine.
+        Example format for query: "data.agent_extension_policy.extensions_to_download"
+        """
+        # This method should never be called if policy is not enabled, this would be a developer error.
+        if not self.is_policy_enforcement_enabled():
+            raise PolicyError("Policy enforcement is disabled, cannot evaluate query.")
+
+        try:
+            full_result = self._engine.eval_query(input_to_check, query)
+            debug_info = "Rule file is located at '{0}'. \nFull query output: {1}".format(self._engine.rule_file, full_result)
+            if full_result is None or full_result == {}:
+                raise PolicyError("query returned empty output. Please validate rule file. {0}".format(debug_info))
+            result = full_result.get('result')
+            if result is None or not isinstance(result, list) or len(result) == 0:
+                raise PolicyError("query returned unexpected output with no 'result' list. Please validate rule file. {0}".format(debug_info))
+            expressions = result[0].get('expressions')
+            if expressions is None or not isinstance(expressions, list) or len(expressions) == 0:
+                raise PolicyError("query returned unexpected output with no 'expressions' list. {0}".format(debug_info))
+            value = expressions[0].get('value')
+            if value is None:
+                raise PolicyError("query returned unexpected output, 'value' not found in 'expressions' list. {0}".format(debug_info))
+            if value == {}:
+                raise PolicyError("query returned expected output format, but value is empty. Please validate policy file '{0}'. '{1}"
+                                  .format(self._engine.policy_file, debug_info))
+                # TODO: surface as a user error with clear instructions for fixing
+            return value
+        except Exception as ex:
+            msg = "Failed to evaluate query for Regorus policy engine: '{0}'".format(ex)
+            self._log_policy(msg=msg, is_success=False)
+            raise PolicyError(msg)
+
+# TODO: Implement class ExtensionPolicyEngine with API is_extension_download_allowed(ext_name) that calls evaluate_query.

azurelinuxagent/ga/policy/regorus.py

@@ -0,0 +1,146 @@
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Requires Python 2.4+ and Openssl 1.0+
+#
+
+import json
+import tempfile
+import azurelinuxagent.common.utils.shellutil as shellutil
+from azurelinuxagent.common.utils.shellutil import CommandError
+from azurelinuxagent.common.exception import AgentError
+
+
+def get_regorus_path():
+    """
+    Returns path to Regorus executable. The executable is not yet officially released as part of the agent package.
+    Currently, the executable is copied into the agent directory for unit testing.
+    """
+    # TODO: update the logic to get regorus path once executable is released as part of agent package
+    # This method is currently mocked for unit tests.
+    regorus_exe = ""
+    return regorus_exe
+
+
+class PolicyError(AgentError):
+    """
+    Error raised during agent policy enforcement.
+    """
+    # TODO: split into two error classes for internal/dev errors and user errors.
+
+
+class Engine:
+    """
+    This class implements the basic operations for the Regorus policy engine via subprocess.
+    Any errors thrown in this class should be caught and handled by PolicyEngine.
+    """
+
+    def __init__(self, policy_file, rule_file):
+        """
+        Rule_file is expected to point to a valid Regorus file.
+
+        Policy_file should be a path to a valid JSON policy (data) file.
+        The expected file format is:
+        {
+            "azureGuestAgentPolicy": {
+                "policyVersion": "0.1.0",
+                "signingRules": {
+                    "extensionSigned": <true, false>
+                },
+                "allowListOnly": <true, false>
+            },
+            "azureGuestExtensionsPolicy": {
+                "allowed_ext_1": {
+                    "signingRules": {
+                        "extensionSigned": <true, false>
+                    }
+                }
+        }
+        """
+        self._engine = None
+        self._rule_file = rule_file
+        self._policy_file = policy_file
+
+
+    @property
+    def policy_file(self):
+        return self._policy_file
+
+    @property
+    def rule_file(self):
+        return self._rule_file
+
+    def eval_query(self, input_to_check, query):
+        """
+        Input_to_check should be type dict.
+        Example format for extension policy:
+        {
+            "extensions": {
+                "<extension_name_1>": {
+                    "signingInfo": {
+                        "extensionSigned": <true, false>
+                    }
+                },
+                "<extension_name_2>": {
+                    "signingInfo": {
+                        "extensionSigned": <true, false>
+                    }
+                }, ...
+        }
+
+        In this method, we call the Regorus executable via run_command to query the policy engine.
+
+        Command:
+            regorus eval -d <rule_file.rego> -d <policy_file.json> -i <input_file.json> <QUERY>
+
+        Parameters:
+            -d, --data <rule.rego|policy.json> : Rego file or JSON file.
+            -i, --input <input.json> : Input file in JSON format.
+            <QUERY>  Query. Rego query block in the format "data.<optional_query>"
+
+        Return Codes:
+            0 - successful query. optional parameters may be missing
+            1 - file error: unsupported file type, error parsing file, file not found
+                ex: "Error: Unsupported data file <file>. Must be rego or json."
+                ex: "Error: Failed to read <file>. No such file or directory."
+            2 - usage error: missing query argument, unexpected or unlabeled parameter
+                ex: "Error: the following required arguments were not provided: <QUERY>"
+                ex: "Error: Unexpected argument <arg> found."
+        """
+        # Write input_to_check to a temp file, because Regorus requires input to be a file path.
+        # Tempfile is automatically cleaned up at the end of with block
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=True) as input_file:
+            json.dump(input_to_check, input_file, indent=4)
+            input_file.flush()
+
+            regorus_exe = get_regorus_path()
+            command = [regorus_exe, "eval", "-d", self._rule_file, "-d", self._policy_file,
+                       "-i", input_file.name, query]
+
+            try:
+                stdout = shellutil.run_command(command)
+            except CommandError as ex:
+                code = ex.returncode
+                if code == 1:
+                    msg = "file error when using policy engine. {0}".format(ex)
+                    # TODO: surface as a user error with clear instructions for fixing
+                elif code == 2:
+                    msg = "incorrect parameters passed to policy engine. {0}".format(ex)
+                    # TODO: log this as an internal/developer error and include debug information
+                else:
+                    msg = "error when using policy engine. {0}".format(ex)
+                raise PolicyError(msg=msg)
+
+            json_output = json.loads(stdout)
+            return json_output

tests/data/policy/agent-extension-data-invalid.json

@@ -0,0 +1,4 @@
+{
+    "policy": {
+        "invalid_attribute": "0"
+    }

tests/data/policy/agent-extension-default-data.json

@@ -0,0 +1,9 @@
+{
+    "azureGuestAgentPolicy": {
+        "policyVersion": "0.1.0",
+        "signingRules": {
+            "extensionSigned": false
+        },
+        "allowListOnly": false
+    }
+}