diff --git a/manifests/v1beta1/components/controller/katib-config.yaml b/manifests/v1beta1/components/controller/katib-config.yaml index e8b92566e1b..d9525a96e3e 100644 --- a/manifests/v1beta1/components/controller/katib-config.yaml +++ b/manifests/v1beta1/components/controller/katib-config.yaml @@ -31,7 +31,7 @@ data: "image": "docker.io/kubeflowkatib/suggestion-hyperopt:latest" }, "grid": { - "image": "docker.io/kubeflowkatib/suggestion-chocolate:latest" + "image": "docker.io/kubeflowkatib/suggestion-optuna:latest" }, "hyperband": { "image": "docker.io/kubeflowkatib/suggestion-hyperband:latest" diff --git a/pkg/suggestion/v1beta1/internal/search_space.py b/pkg/suggestion/v1beta1/internal/search_space.py index 05efc3efb23..25d68761805 100644 --- a/pkg/suggestion/v1beta1/internal/search_space.py +++ b/pkg/suggestion/v1beta1/internal/search_space.py @@ -13,11 +13,12 @@ # limitations under the License. import logging +import numpy as np from pkg.apis.manager.v1beta1.python import api_pb2 as api +from pkg.suggestion.v1beta1.internal.constant import INTEGER, DOUBLE, CATEGORICAL, DISCRETE import pkg.suggestion.v1beta1.internal.constant as constant - logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -36,15 +37,38 @@ def convert(experiment): search_space.goal = constant.MIN_GOAL for p in experiment.spec.parameter_specs.parameters: search_space.params.append( - HyperParameterSearchSpace.convertParameter(p)) + HyperParameterSearchSpace.convert_parameter(p)) return search_space + @staticmethod + def convert_to_combinations(search_space): + combinations = {} + + for parameter in search_space.params: + if parameter.type == INTEGER: + combinations[parameter.name] = range(int(parameter.min), int(parameter.max)+1, int(parameter.step)) + elif parameter.type == DOUBLE: + if parameter.step == "" or parameter.step is None: + raise Exception( + "Param {} step is nil; For discrete search space, all parameters must include step". + format(parameter.name) + ) + double_list = np.arange(float(parameter.min), float(parameter.max)+float(parameter.step), + float(parameter.step)) + if double_list[-1] > float(parameter.max): + double_list = double_list[:-1] + combinations[parameter.name] = double_list + elif parameter.type == CATEGORICAL or parameter.type == DISCRETE: + combinations[parameter.name] = parameter.list + + return combinations + def __str__(self): return "HyperParameterSearchSpace(goal: {}, ".format(self.goal) + \ "params: {})".format(", ".join([element.__str__() for element in self.params])) @staticmethod - def convertParameter(p): + def convert_parameter(p): if p.parameter_type == api.INT: # Default value for INT parameter step is 1 step = 1 diff --git a/pkg/suggestion/v1beta1/optuna/base_service.py b/pkg/suggestion/v1beta1/optuna/base_service.py index 3943c4c0820..c4c742514f2 100644 --- a/pkg/suggestion/v1beta1/optuna/base_service.py +++ b/pkg/suggestion/v1beta1/optuna/base_service.py @@ -17,6 +17,7 @@ from pkg.suggestion.v1beta1.internal.constant import INTEGER, DOUBLE, CATEGORICAL, DISCRETE, MAX_GOAL from pkg.suggestion.v1beta1.internal.trial import Assignment +from pkg.suggestion.v1beta1.internal.search_space import HyperParameterSearchSpace class BaseOptunaService(object): @@ -48,6 +49,10 @@ def _create_sampler(self): elif self.algorithm_name == "random": return optuna.samplers.RandomSampler(**self.algorithm_config) + elif self.algorithm_name == "grid": + combinations = HyperParameterSearchSpace.convert_to_combinations(self.search_space) + return optuna.samplers.GridSampler(combinations, **self.algorithm_config) + def get_suggestions(self, trials, current_request_number): if len(trials) != 0: self._tell(trials) diff --git a/pkg/suggestion/v1beta1/optuna/service.py b/pkg/suggestion/v1beta1/optuna/service.py index 9350e27ebb7..0df00de45cc 100644 --- a/pkg/suggestion/v1beta1/optuna/service.py +++ b/pkg/suggestion/v1beta1/optuna/service.py @@ -15,10 +15,10 @@ import threading import grpc import logging +import itertools from pkg.apis.manager.v1beta1.python import api_pb2 from pkg.apis.manager.v1beta1.python import api_pb2_grpc -from pkg.suggestion.v1beta1.internal.constant import INTEGER, DOUBLE from pkg.suggestion.v1beta1.internal.search_space import HyperParameterSearchSpace from pkg.suggestion.v1beta1.internal.trial import Trial, Assignment from pkg.suggestion.v1beta1.optuna.base_service import BaseOptunaService @@ -55,7 +55,7 @@ def GetSuggestions(self, request, context): def ValidateAlgorithmSettings(self, request, context): is_valid, message = OptimizerConfiguration.validate_algorithm_spec( - request.experiment.spec.algorithm) + request.experiment) if not is_valid: context.set_code(grpc.StatusCode.INVALID_ARGUMENT) context.set_details(message) @@ -86,6 +86,9 @@ class OptimizerConfiguration(object): "random": { "seed": lambda x: int(x), }, + "grid": { + "seed": lambda x: int(x), + } } @classmethod @@ -110,7 +113,8 @@ def convert_algorithm_spec(cls, algorithm_spec): return algorithm_spec.algorithm_name, config @classmethod - def validate_algorithm_spec(cls, algorithm_spec): + def validate_algorithm_spec(cls, experiment): + algorithm_spec = experiment.spec.algorithm algorithm_name = algorithm_spec.algorithm_name algorithm_settings = algorithm_spec.algorithm_settings @@ -120,6 +124,10 @@ def validate_algorithm_spec(cls, algorithm_spec): return cls._validate_cmaes_setting(algorithm_settings) elif algorithm_name == "random": return cls._validate_random_setting(algorithm_settings) + elif algorithm_name == "grid": + return cls._validate_grid_setting(experiment) + else: + return False, "unknown algorithm name {}".format(algorithm_name) @classmethod def _validate_tpe_setting(cls, algorithm_spec): @@ -178,3 +186,34 @@ def _validate_random_setting(cls, algorithm_settings): exception=e) return True, "" + + @classmethod + def _validate_grid_setting(cls, experiment): + algorithm_settings = experiment.spec.algorithm.algorithm_settings + search_space = HyperParameterSearchSpace.convert(experiment) + + for s in algorithm_settings: + try: + if s.name == "random_state": + if not int(s.value) >= 0: + return False, "" + else: + return False, "unknown setting {} for algorithm grid".format(s.name) + + except Exception as e: + return False, "failed to validate {name}({value}): {exception}".format(name=s.name, value=s.value, + exception=e) + + try: + combinations = HyperParameterSearchSpace.convert_to_combinations(search_space) + num_combinations = len(list(itertools.product(*combinations.values()))) + max_trial_count = experiment.spec.max_trial_count + if max_trial_count > num_combinations: + return False, "Max Trial Count: {max_trial} > all possible search combinations: {combinations}".\ + format(max_trial=max_trial_count, combinations=num_combinations) + + except Exception as e: + return False, "failed to validate parameters({parameters}): {exception}".\ + format(parameters=search_space.params, exception=e) + + return True, "" diff --git a/test/unit/v1beta1/suggestion/test_optuna_service.py b/test/unit/v1beta1/suggestion/test_optuna_service.py index b74d0afe9b1..9b6c08115ff 100644 --- a/test/unit/v1beta1/suggestion/test_optuna_service.py +++ b/test/unit/v1beta1/suggestion/test_optuna_service.py @@ -39,6 +39,7 @@ def setup_method(self): ["multivariate-tpe", {"n_startup_trials": "20", "n_ei_candidates": "10", "random_state": "71"}], ["cmaes", {"restart_strategy": "ipop", "sigma": "2", "random_state": "71"}], ["random", {"random_state": "71"}], + ["grid", {"random_state": "71"}], ], ) def test_get_suggestion(self, algorithm_name, algorithm_settings): @@ -82,7 +83,7 @@ def test_get_suggestion(self, algorithm_name, algorithm_settings): name="param-4", parameter_type=api_pb2.DOUBLE, feasible_space=api_pb2.FeasibleSpace( - max="5", min="1", list=[]) + max="5", min="1", step="1", list=[]) ) ] ) @@ -184,280 +185,99 @@ def test_get_suggestion(self, algorithm_name, algorithm_settings): assert code == grpc.StatusCode.OK assert 2 == len(response.parameter_assignments) - def test_validate_algorithm_settings(self): - # Invalid algorithm name - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="invalid", - ), - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.UNKNOWN - - # [TPE] Valid Case - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="tpe", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="n_startup_trials", - value="5", - ), - api_pb2.AlgorithmSetting( - name="n_ei_candidates", - value="24", - ), - api_pb2.AlgorithmSetting( - name="random_state", - value="1", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.OK - - # [TPE] Invalid Parameter Name - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="tpe", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="invalid", - value="5", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.INVALID_ARGUMENT - - # [TPE] Invalid n_startup_trials - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="tpe", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="n_startup_trials", - value="-1", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.INVALID_ARGUMENT - - # [TPE] Invalid n_ei_candidates - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="tpe", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="n_ei_candidates", - value="-1", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.INVALID_ARGUMENT - - # [TPE] Invalid random_state - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="tpe", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="random_state", - value="-1", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.INVALID_ARGUMENT - - # [Multivariate-TPE] Valid Case - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="multivariate-tpe", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="n_startup_trials", - value="5", - ), - api_pb2.AlgorithmSetting( - name="n_ei_candidates", - value="24", - ), - api_pb2.AlgorithmSetting( - name="random_state", - value="1", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.OK - - # [CMAES] Valid Case - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="cmaes", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="restart_strategy", - value="ipop", - ), - api_pb2.AlgorithmSetting( - name="sigma", - value="0.1", - ), - api_pb2.AlgorithmSetting( - name="random_state", - value="10", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.OK - - # [CMAES] Invalid parameter name - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="cmaes", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="invalid", - value="invalid", - ), - api_pb2.AlgorithmSetting( - name="sigma", - value="0.1", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.INVALID_ARGUMENT - - # [CMAES] Invalid restart_strategy - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="cmaes", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="restart_strategy", - value="invalid", - ), - api_pb2.AlgorithmSetting( - name="sigma", - value="0.1", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.INVALID_ARGUMENT - - # [CMAES] Invalid sigma - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="cmaes", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="restart_strategy", - value="None", - ), - api_pb2.AlgorithmSetting( - name="sigma", - value="-10", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.INVALID_ARGUMENT - - # [CMAES] Invalid random_state - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="cmaes", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="sigma", - value="0.2", - ), - api_pb2.AlgorithmSetting( - name="random_state", - value="-20", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.INVALID_ARGUMENT - - # [CMAES] Invalid number of parameters - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="cmaes", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="sigma", - value="0.2", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.INVALID_ARGUMENT - - # [RANDOM] Valid Case - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="random", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="random_state", - value="10", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.OK - - # [RANDOM] Invalid parameter name - experiment_spec = api_pb2.ExperimentSpec( - algorithm=api_pb2.AlgorithmSpec( - algorithm_name="random", - algorithm_settings=[ - api_pb2.AlgorithmSetting( - name="invalid", - value="invalid", - ), - ], - ) - ) - _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.INVALID_ARGUMENT - - # [RANDOM] Invalid random_state + @pytest.mark.parametrize( + ["algorithm_name", "algorithm_settings", "max_trial_count", "parameters", "result"], + [ + # Invalid algorithm name + ["invalid", {}, 1, [], grpc.StatusCode.INVALID_ARGUMENT], + + # [TPE] Valid case + ["tpe", {"n_startup_trials": "5", "n_ei_candidates": "24", "random_state": "1"}, 100, [], + grpc.StatusCode.OK], + # [TPE] Invalid parameter name + ["tpe", {"invalid": "5"}, 100, [], grpc.StatusCode.INVALID_ARGUMENT], + # [TPE] Invalid n_startup_trials + ["tpe", {"n_startup_trials": "-1"}, 100, [], grpc.StatusCode.INVALID_ARGUMENT], + # [TPE] Invalid n_ei_candidate + ["tpe", {"n_ei_candidate": "-1"}, 100, [], grpc.StatusCode.INVALID_ARGUMENT], + # [TPE] Invalid random_state + ["tpe", {"random_state": "-1"}, 100, [], grpc.StatusCode.INVALID_ARGUMENT], + + # [Multivariate-TPE] Valid case + ["multivariate-tpe", {"n_startup_trials": "5", "n_ei_candidates": "24", "random_state": "1"}, 100, [], + grpc.StatusCode.OK], + + # [CMAES] Valid case + ["cmaes", {"restart_strategy": "ipop", "sigma": "0.1", "random_state": "10"}, 20, [], grpc.StatusCode.OK], + # [CMAES] Invalid parameter name + ["cmaes", {"invalid": "invalid", "sigma": "0.1"}, 100, [], grpc.StatusCode.INVALID_ARGUMENT], + # [CMAES] Invalid restart_strategy + ["cmaes", {"restart_strategy": "invalid", "sigma": "0.1"}, 15, [], grpc.StatusCode.INVALID_ARGUMENT], + # [CMAES] Invalid sigma + ["cmaes", {"restart_strategy": "None", "sigma": "-10"}, 55, [], grpc.StatusCode.INVALID_ARGUMENT], + # [CMAES] Invalid random_state + ["cmaes", {"sigma": "0.2", "random_state": "-20"}, 25, [], grpc.StatusCode.INVALID_ARGUMENT], + # [CMAES] Invalid number of parameters + ["cmaes", {"sigma": "0.2"}, 5, [], grpc.StatusCode.INVALID_ARGUMENT], + + # [RANDOM] Valid Case + ["random", {"random_state": "10"}, 23, [], grpc.StatusCode.OK], + # [RANDOM] Invalid parameter name + ["random", {"invalid": "invalid"}, 33, [], grpc.StatusCode.INVALID_ARGUMENT], + # [RANDOM] Invalid random_state + ["random", {"random_state": "-1"}, 33, [], grpc.StatusCode.INVALID_ARGUMENT], + + # [GRID] Valid Case + ["grid", {"random_state": "10"}, 5, + [{"name": "param-1", + "type": api_pb2.INT, + "feasible_space": api_pb2.FeasibleSpace(max="5", min="1", list=[])}, + ], grpc.StatusCode.OK], + # [GRID] Invalid parameter name + ["grid", {"invalid": "invalid"}, 33, [], grpc.StatusCode.INVALID_ARGUMENT], + # [GRID] Invalid random_state + ["grid", {"random_state": "-1"}, 10, [], grpc.StatusCode.INVALID_ARGUMENT], + # [GRID] Invalid feasible_space + ["grid", {"random_state": "1"}, 26, + [{"name": "param-1", + "type": api_pb2.DOUBLE, + "feasible_space": api_pb2.FeasibleSpace(max="5", min="1", list=[])}, + ], grpc.StatusCode.INVALID_ARGUMENT], + # [GRID] Invalid max_trial_count + ["grid", {"random_state": "1"}, 26, + [{"name": "param-1", + "type": api_pb2.INT, + "feasible_space": api_pb2.FeasibleSpace(max="5", min="1", list=[])}, + {"name": "param-2", + "type": api_pb2.DOUBLE, + "feasible_space": api_pb2.FeasibleSpace(max="5", min="1", step="1", list=[])}, + ], grpc.StatusCode.INVALID_ARGUMENT], + ], + ) + def test_validate_algorithm_settings(self, algorithm_name, algorithm_settings, max_trial_count, parameters, result): experiment_spec = api_pb2.ExperimentSpec( + max_trial_count=max_trial_count, algorithm=api_pb2.AlgorithmSpec( - algorithm_name="random", + algorithm_name=algorithm_name, algorithm_settings=[ api_pb2.AlgorithmSetting( - name="random_state", - value="-1", - ), + name=name, + value=value + ) for name, value in algorithm_settings.items() ], + ), + parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs( + parameters=[ + api_pb2.ParameterSpec( + name=param["name"], + parameter_type=param["type"], + feasible_space=param["feasible_space"], + ) for param in parameters + ] ) ) _, _, code, _ = utils.call_validate(self.test_server, experiment_spec) - assert code == grpc.StatusCode.INVALID_ARGUMENT + assert code == result if __name__ == '__main__':