docker.io/kubeflowkatib/suggestion-skopt
diff --git a/pkg/suggestion/v1beta1/chocolate/__init__.py b/pkg/suggestion/v1beta1/chocolate/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/pkg/suggestion/v1beta1/chocolate/base_service.py b/pkg/suggestion/v1beta1/chocolate/base_service.py
deleted file mode 100644
index 8f9c0613ca9..00000000000
--- a/pkg/suggestion/v1beta1/chocolate/base_service.py
+++ /dev/null
@@ -1,234 +0,0 @@
-# Copyright 2022 The Kubeflow Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import chocolate as choco
-import logging
-import base64
-import warnings
-
-from pkg.suggestion.v1beta1.internal.constant import MAX_GOAL, INTEGER, DOUBLE, CATEGORICAL, DISCRETE
-from pkg.suggestion.v1beta1.internal.trial import Assignment
-
-logger = logging.getLogger(__name__)
-
-DB_ADDRESS = "sqlite:///my_db.db?check_same_thread=False"
-DB_FIELD_LOSS = "_loss"
-DB_FIELD_CHOCOLATE_ID = "_chocolate_id"
-DB_FIELD_TRIAL_NAME = "_trial_name"
-
-DEPRECATED_ALGORITHM_NAME = {
- "chocolate-random": "random",
- "chocolate-quasirandom": "quasirandom",
- "chocolate-bayesian-optimization": "bayesianoptimization",
- "chocolate-mocmaes": "mocmaes",
-}
-
-
-class BaseChocolateService(object):
- """
- Refer to https://chocolate.readthedocs.io/
- """
-
- def __init__(self, algorithm_name, search_space):
- self.conn = choco.SQLiteConnection(DB_ADDRESS)
- self.search_space = search_space
- self.chocolate_optimizer = None
- self.create_optimizer(algorithm_name)
- # created_trials is the list of dicts with all created trials assignments, loss and trial name
- # _chocolate_id is the ID of the trial, Assignment names are encoded,
- # _loss is the target metric, _trial_name is the Trial name
- # One row example:
- # {'_chocolate_id': 0, 'LS1scg==': 0.001, 'LS1udW0tZXBvY2hz': 1, 'LS1udW0tbGF5ZXJz': 2,
- # "_loss": "0.97", "_trial_name": "grid-hsdvfdwl"}
- self.created_trials = []
- self.recorded_trials_names = []
-
- def create_optimizer(self, algorithm_name):
-
- # Search Space example: {"x" : choco.uniform(-6, 6), "y" : choco.uniform(-6, 6)}
- chocolate_search_space = {}
-
- for param in self.search_space.params:
- key = BaseChocolateService.encode(param.name)
- # Chocolate quantized_uniform distribution uses half-open interval: [low, high).
- if param.type == INTEGER:
- chocolate_search_space[key] = choco.quantized_uniform(
- int(param.min), int(param.max) + int(param.step), int(param.step))
- elif param.type == DOUBLE:
- chocolate_search_space[key] = choco.quantized_uniform(
- float(param.min), float(param.max) + float(param.step), float(param.step))
- # For Categorical and Discrete insert indexes to DB from list of values
- elif param.type == CATEGORICAL or param.type == DISCRETE:
- chocolate_search_space[key] = choco.choice(
- [idx for idx, _ in enumerate(param.list)])
-
- if algorithm_name in DEPRECATED_ALGORITHM_NAME:
- warnings.warn(
- "Algorithm name '{}' is deprecated. Please use '{}'.".format(
- algorithm_name, DEPRECATED_ALGORITHM_NAME[algorithm_name],
- ),
- DeprecationWarning,
- )
- algorithm_name = DEPRECATED_ALGORITHM_NAME[algorithm_name]
-
- # Refer to https://chocolate.readthedocs.io/tutorials/algo.html
- if algorithm_name == "grid":
- self.chocolate_optimizer = choco.Grid(
- self.conn, chocolate_search_space, clear_db=True)
- # hyperopt-random is the default option in katib.
- elif algorithm_name == "random":
- self.chocolate_optimizer = choco.Random(
- self.conn, chocolate_search_space, clear_db=True)
- elif algorithm_name == "quasirandom":
- self.chocolate_optimizer = choco.QuasiRandom(
- self.conn, chocolate_search_space, clear_db=True)
- elif algorithm_name == "bayesianoptimization":
- self.chocolate_optimizer = choco.Bayes(
- self.conn, chocolate_search_space, clear_db=True)
- # elif self.algorithm_name == "chocolate-CMAES":
- # self.chocolate_optimizer = choco.CMAES(self.conn, chocolate_search_space, clear_db=True)
- elif algorithm_name == "mocmaes":
- mu = 1
- self.chocolate_optimizer = choco.MOCMAES(
- self.conn, chocolate_search_space, mu=mu, clear_db=True)
- else:
- raise Exception(
- '"Failed to create Chocolate optimizer for the algorithm: {}'.format(algorithm_name))
-
- def getSuggestions(self, trials, current_request_number, total_request_number):
- """
- Get the new suggested trials with chocolate algorithm.
- """
- logger.info("-" * 100 + "\n")
- logger.info("New GetSuggestions call with total requested {} and currently requesting {} \n".format(
- total_request_number, current_request_number))
- for _, trial in enumerate(trials):
- if trial.name not in self.recorded_trials_names:
- loss_for_choco = float(trial.target_metric.value)
- if self.search_space.goal == MAX_GOAL:
- loss_for_choco = -1 * loss_for_choco
-
- trial_assignments_dict = {}
- for param in self.search_space.params:
- param_assignment = None
- for assignment in trial.assignments:
- if param.name == assignment.name:
- param_assignment = assignment.value
- break
- if param.type == INTEGER:
- param_assignment = int(param_assignment)
- elif param.type == DOUBLE:
- param_assignment = float(param_assignment)
- elif param.type == CATEGORICAL or param.type == DISCRETE:
- param_assignment = param.list.index(param_assignment)
- trial_assignments_dict.update({BaseChocolateService.encode(
- param.name): param_assignment})
-
- # Finding index for the current Trial Assignments in created_trial list without loss
- new_trial_loss_idx = -1
- i = 0
- while new_trial_loss_idx == -1 and i < len(self.created_trials):
- # Created Trial must not include loss and must have the same param assignment
- if ((DB_FIELD_LOSS not in self.created_trials[i] or
- self.created_trials[i][DB_FIELD_LOSS] is None) and
- len(trial_assignments_dict.items() & self.created_trials[i].items()) ==
- len(self.search_space.params)):
- new_trial_loss_idx = i
- i += 1
-
- if new_trial_loss_idx != -1:
- self.created_trials[new_trial_loss_idx][DB_FIELD_LOSS] = loss_for_choco
- self.created_trials[new_trial_loss_idx][DB_FIELD_TRIAL_NAME] = trial.name
-
- # Update sqlite database with new loss and trial assignments
- id_filter = {
- DB_FIELD_CHOCOLATE_ID: self.created_trials[new_trial_loss_idx][DB_FIELD_CHOCOLATE_ID]}
- self.conn.update_result(
- id_filter,
- self.created_trials[new_trial_loss_idx])
-
- self.recorded_trials_names.append(trial.name)
-
- logger.info("New record in sqlite DB is updated")
- logger.info("{}\n".format(
- self.created_trials[new_trial_loss_idx]))
- # Assuming that created_trials are already populated
- # TODO: Handle Restart of algorithm pod
- logger.info("{} Trials created in DB".format(len(self.created_trials)))
- if total_request_number != len(self.created_trials) + current_request_number:
- logger.info("Mismatch in generated trials with k8s suggestions trials")
- new_actual_requested_no = total_request_number - len(self.created_trials)
- prev_generated_no = current_request_number - new_actual_requested_no
- logger.info(
- "In this call, New {} Trials will be generated, {} Trials will be reused from previously generated".format(
- new_actual_requested_no, prev_generated_no))
-
- list_of_assignments = []
- if prev_generated_no > 0:
- for params in self.created_trials[-prev_generated_no:]:
- if DB_FIELD_TRIAL_NAME in params:
- logger.error("Trial already updated in selected assignment {}".format(params))
- new_assignment = BaseChocolateService.convert(
- self.search_space, params)
- list_of_assignments.append(new_assignment)
-
- for i in range(new_actual_requested_no):
- try:
- token, chocolate_params = self.chocolate_optimizer.next()
- new_assignment = BaseChocolateService.convert(
- self.search_space, chocolate_params)
- list_of_assignments.append(new_assignment)
- logger.info("New suggested parameters for Trial with chocolate_id: {}".format(
- token[DB_FIELD_CHOCOLATE_ID]))
- for assignment in new_assignment:
- logger.info("Name = {}, Value = {}".format(
- assignment.name, assignment.value))
- logger.info("-" * 50 + "\n")
- # Add new trial assignment with chocolate_id to created trials
- token.update(chocolate_params)
- new_trial_dict = token
- self.created_trials.append(new_trial_dict)
-
- except StopIteration:
- logger.info(
- "Chocolate db is exhausted, increase Search Space or decrease maxTrialCount!")
-
- if len(list_of_assignments) > 0:
- logger.info("GetSuggestions returns {} Trials from requested {} Trials\n\n".format(
- len(list_of_assignments), current_request_number))
-
- return list_of_assignments
-
- @staticmethod
- def convert(search_space, chocolate_params):
- assignments = []
- for param in search_space.params:
- key = BaseChocolateService.encode(param.name)
- if param.type == INTEGER:
- assignments.append(Assignment(
- param.name, chocolate_params[key]))
- elif param.type == DOUBLE:
- assignments.append(Assignment(
- param.name, chocolate_params[key]))
- elif param.type == CATEGORICAL or param.type == DISCRETE:
- assignments.append(Assignment(
- param.name, param.list[chocolate_params[key]]))
- return assignments
-
- @staticmethod
- def encode(name):
- """Encode the name. Chocolate will check if the name contains hyphens.
- Thus we need to encode it.
- """
- return base64.b64encode(name.encode('utf-8')).decode('utf-8')
diff --git a/pkg/suggestion/v1beta1/chocolate/service.py b/pkg/suggestion/v1beta1/chocolate/service.py
deleted file mode 100644
index 9ab1b77e59d..00000000000
--- a/pkg/suggestion/v1beta1/chocolate/service.py
+++ /dev/null
@@ -1,96 +0,0 @@
-# Copyright 2022 The Kubeflow Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-import grpc
-
-from pkg.apis.manager.v1beta1.python import api_pb2
-from pkg.apis.manager.v1beta1.python import api_pb2_grpc
-
-from pkg.suggestion.v1beta1.internal.constant import INTEGER, DOUBLE, CATEGORICAL, DISCRETE
-from pkg.suggestion.v1beta1.internal.search_space import HyperParameterSearchSpace
-from pkg.suggestion.v1beta1.internal.trial import Trial, Assignment
-from pkg.suggestion.v1beta1.chocolate.base_service import BaseChocolateService
-from pkg.suggestion.v1beta1.internal.base_health_service import HealthServicer
-
-import numpy as np
-import itertools
-
-logger = logging.getLogger(__name__)
-
-
-class ChocolateService(api_pb2_grpc.SuggestionServicer, HealthServicer):
- def __init__(self):
- super(ChocolateService, self).__init__()
- self.base_service = None
- self.is_first_run = True
-
- def ValidateAlgorithmSettings(self, request, context):
- algorithm_name = request.experiment.spec.algorithm.algorithm_name
- if algorithm_name == "grid":
- search_space = HyperParameterSearchSpace.convert(
- request.experiment)
- available_space = {}
- for param in search_space.params:
- if param.type == INTEGER:
- available_space[param.name] = range(int(param.min), int(param.max)+1, int(param.step))
-
- elif param.type == DOUBLE:
- if param.step == "" or param.step is None:
- return self._set_validate_context_error(
- context, "Param: {} step is nil".format(param.name))
- double_list = np.arange(float(param.min), float(param.max)+float(param.step), float(param.step))
- if double_list[-1] > float(param.max):
- double_list = double_list[:-1]
- available_space[param.name] = double_list
-
- elif param.type == CATEGORICAL or param.type == DISCRETE:
- available_space[param.name] = param.list
-
- num_combinations = len(list(itertools.product(*available_space.values())))
- max_trial_count = request.experiment.spec.max_trial_count
-
- if max_trial_count > num_combinations:
- return self._set_validate_context_error(
- context, "Max Trial Count: {} > all possible search space combinations: {}".format(
- max_trial_count, num_combinations)
- )
-
- return api_pb2.ValidateAlgorithmSettingsReply()
-
- def GetSuggestions(self, request, context):
- """
- Main function to provide suggestion.
- """
-
- if self.is_first_run:
- search_space = HyperParameterSearchSpace.convert(
- request.experiment)
- self.base_service = BaseChocolateService(
- algorithm_name=request.experiment.spec.algorithm.algorithm_name,
- search_space=search_space)
- self.is_first_run = False
-
- trials = Trial.convert(request.trials)
- new_assignments = self.base_service.getSuggestions(
- trials, request.current_request_number, request.total_request_number)
- return api_pb2.GetSuggestionsReply(
- parameter_assignments=Assignment.generate(new_assignments)
- )
-
- def _set_validate_context_error(self, context, error_message):
- context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
- context.set_details(error_message)
- logger.info(error_message)
- return api_pb2.ValidateAlgorithmSettingsReply()
diff --git a/scripts/v1beta1/build.sh b/scripts/v1beta1/build.sh
index 5e3c62b0122..3953f49f54d 100755
--- a/scripts/v1beta1/build.sh
+++ b/scripts/v1beta1/build.sh
@@ -84,9 +84,6 @@ echo -e "\nBuilding suggestion images..."
echo -e "\nBuilding hyperopt suggestion...\n"
docker buildx build --platform "linux/${ARCH}" -t "${REGISTRY}/suggestion-hyperopt:${TAG}" -f ${CMD_PREFIX}/suggestion/hyperopt/${VERSION}/Dockerfile .
-echo -e "\nBuilding chocolate suggestion...\n"
-docker buildx build --platform "linux/${ARCH}" -t "${REGISTRY}/suggestion-chocolate:${TAG}" -f ${CMD_PREFIX}/suggestion/chocolate/${VERSION}/Dockerfile .
-
echo -e "\nBuilding hyperband suggestion...\n"
docker buildx build --platform "linux/${ARCH}" -t "${REGISTRY}/suggestion-hyperband:${TAG}" -f ${CMD_PREFIX}/suggestion/hyperband/${VERSION}/Dockerfile .
diff --git a/scripts/v1beta1/push.sh b/scripts/v1beta1/push.sh
index 95f5d3c98eb..6f0627b4081 100755
--- a/scripts/v1beta1/push.sh
+++ b/scripts/v1beta1/push.sh
@@ -59,9 +59,6 @@ echo -e "\nPushing suggestion images..."
echo -e "\nPushing hyperopt suggestion...\n"
docker push "${REGISTRY}/suggestion-hyperopt:${TAG}"
-echo -e "\nPushing chocolate suggestion...\n"
-docker push "${REGISTRY}/suggestion-chocolate:${TAG}"
-
echo -e "\nPushing hyperband suggestion...\n"
docker push "${REGISTRY}/suggestion-hyperband:${TAG}"
diff --git a/test/e2e/v1beta1/hack/aws/argo_workflow.py b/test/e2e/v1beta1/hack/aws/argo_workflow.py
index 7064273ff06..36a403d7d12 100644
--- a/test/e2e/v1beta1/hack/aws/argo_workflow.py
+++ b/test/e2e/v1beta1/hack/aws/argo_workflow.py
@@ -49,7 +49,6 @@
"file-metrics-collector": "cmd/metricscollector/v1beta1/file-metricscollector/Dockerfile",
"tfevent-metrics-collector": "cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile",
"suggestion-hyperopt": "cmd/suggestion/hyperopt/v1beta1/Dockerfile",
- "suggestion-chocolate": "cmd/suggestion/chocolate/v1beta1/Dockerfile",
"suggestion-skopt": "cmd/suggestion/skopt/v1beta1/Dockerfile",
"suggestion-hyperband": "cmd/suggestion/hyperband/v1beta1/Dockerfile",
"suggestion-goptuna": "cmd/suggestion/goptuna/v1beta1/Dockerfile",
diff --git a/test/e2e/v1beta1/scripts/gh-actions/build-load.sh b/test/e2e/v1beta1/scripts/gh-actions/build-load.sh
index 30dc2f2f159..6259d5761dd 100755
--- a/test/e2e/v1beta1/scripts/gh-actions/build-load.sh
+++ b/test/e2e/v1beta1/scripts/gh-actions/build-load.sh
@@ -158,7 +158,6 @@ cleanup_build_cache
# Suggestion images
echo -e "\nBuilding suggestion images..."
run "suggestion-hyperopt" "$CMD_PREFIX/suggestion/hyperopt/$VERSION/Dockerfile"
-run "suggestion-chocolate" "$CMD_PREFIX/suggestion/chocolate/$VERSION/Dockerfile"
run "suggestion-hyperband" "$CMD_PREFIX/suggestion/hyperband/$VERSION/Dockerfile"
run "suggestion-skopt" "$CMD_PREFIX/suggestion/skopt/$VERSION/Dockerfile"
run "suggestion-goptuna" "$CMD_PREFIX/suggestion/goptuna/$VERSION/Dockerfile"
diff --git a/test/unit/v1beta1/suggestion/test_chocolate_service.py b/test/unit/v1beta1/suggestion/test_chocolate_service.py
deleted file mode 100644
index 34809f4863f..00000000000
--- a/test/unit/v1beta1/suggestion/test_chocolate_service.py
+++ /dev/null
@@ -1,300 +0,0 @@
-# Copyright 2022 The Kubeflow Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-import grpc
-import grpc_testing
-import unittest
-import pytest
-
-from pkg.apis.manager.v1beta1.python import api_pb2
-
-from pkg.suggestion.v1beta1.chocolate.service import ChocolateService
-
-import utils
-
-
-class TestChocolate(unittest.TestCase):
- def setUp(self):
- servicers = {
- api_pb2.DESCRIPTOR.services_by_name['Suggestion']: ChocolateService(
- )
- }
-
- self.test_server = grpc_testing.server_from_dictionary(
- servicers, grpc_testing.strict_real_time())
-
- def test_get_suggestion(self):
- trials = [
- api_pb2.Trial(
- name="test-asfjh",
- spec=api_pb2.TrialSpec(
- objective=api_pb2.ObjectiveSpec(
- type=api_pb2.MAXIMIZE,
- objective_metric_name="metric-2",
- goal=0.9
- ),
- parameter_assignments=api_pb2.TrialSpec.ParameterAssignments(
- assignments=[
- api_pb2.ParameterAssignment(
- name="param-1",
- value="2",
- ),
- api_pb2.ParameterAssignment(
- name="param-2",
- value="cat1",
- ),
- api_pb2.ParameterAssignment(
- name="param-3",
- value="2",
- ),
- api_pb2.ParameterAssignment(
- name="param-4",
- value="3.44",
- )
- ]
- )
- ),
- status=api_pb2.TrialStatus(
- observation=api_pb2.Observation(
- metrics=[
- api_pb2.Metric(
- name="metric=1",
- value="435"
- ),
- api_pb2.Metric(
- name="metric=2",
- value="5643"
- ),
- ]
- )
- )
- ),
- api_pb2.Trial(
- name="test-234hs",
- spec=api_pb2.TrialSpec(
- objective=api_pb2.ObjectiveSpec(
- type=api_pb2.MAXIMIZE,
- objective_metric_name="metric-2",
- goal=0.9
- ),
- parameter_assignments=api_pb2.TrialSpec.ParameterAssignments(
- assignments=[
- api_pb2.ParameterAssignment(
- name="param-1",
- value="3",
- ),
- api_pb2.ParameterAssignment(
- name="param-2",
- value="cat2",
- ),
- api_pb2.ParameterAssignment(
- name="param-3",
- value="6",
- ),
- api_pb2.ParameterAssignment(
- name="param-4",
- value="4.44",
- )
- ]
- )
- ),
- status=api_pb2.TrialStatus(
- observation=api_pb2.Observation(
- metrics=[
- api_pb2.Metric(
- name="metric=1",
- value="123"
- ),
- api_pb2.Metric(
- name="metric=2",
- value="3028"
- ),
- ]
- )
- )
- )
- ]
- experiment = api_pb2.Experiment(
- name="test",
- spec=api_pb2.ExperimentSpec(
- algorithm=api_pb2.AlgorithmSpec(
- algorithm_name="grid",
- ),
- objective=api_pb2.ObjectiveSpec(
- type=api_pb2.MAXIMIZE,
- goal=0.9
- ),
- parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
- parameters=[
- api_pb2.ParameterSpec(
- name="param-1",
- parameter_type=api_pb2.INT,
- feasible_space=api_pb2.FeasibleSpace(
- max="5", min="1", list=[]),
- ),
- api_pb2.ParameterSpec(
- name="param-2",
- parameter_type=api_pb2.CATEGORICAL,
- feasible_space=api_pb2.FeasibleSpace(
- max=None, min=None, list=["cat1", "cat2", "cat3"])
- ),
- api_pb2.ParameterSpec(
- name="param-3",
- parameter_type=api_pb2.DISCRETE,
- feasible_space=api_pb2.FeasibleSpace(
- max=None, min=None, list=["3", "2", "6"])
- ),
- api_pb2.ParameterSpec(
- name="param-4",
- parameter_type=api_pb2.DOUBLE,
- feasible_space=api_pb2.FeasibleSpace(
- max="5", min="1", list=[], step="0.5")
- )
- ]
- )
- )
- )
-
- request = api_pb2.GetSuggestionsRequest(
- experiment=experiment,
- trials=trials,
- current_request_number=2,
- total_request_number=2,
- )
-
- get_suggestion = self.test_server.invoke_unary_unary(
- method_descriptor=(api_pb2.DESCRIPTOR
- .services_by_name['Suggestion']
- .methods_by_name['GetSuggestions']),
- invocation_metadata={},
- request=request, timeout=100)
-
- response, metadata, code, details = get_suggestion.termination()
- print(response.parameter_assignments)
- self.assertEqual(code, grpc.StatusCode.OK)
- self.assertEqual(2, len(response.parameter_assignments))
-
- def test_validate_algorithm_settings(self):
- # Valid case.
- experiment_spec = api_pb2.ExperimentSpec(
- algorithm=api_pb2.AlgorithmSpec(
- algorithm_name="grid",
- ),
- parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
- parameters=[
- api_pb2.ParameterSpec(
- name="param-1",
- parameter_type=api_pb2.INT,
- feasible_space=api_pb2.FeasibleSpace(
- max="5", min="1", list=[]),
- ),
- api_pb2.ParameterSpec(
- name="param-2",
- parameter_type=api_pb2.CATEGORICAL,
- feasible_space=api_pb2.FeasibleSpace(
- max=None, min=None, list=["cat1", "cat2", "cat3"])
- ),
- api_pb2.ParameterSpec(
- name="param-3",
- parameter_type=api_pb2.DISCRETE,
- feasible_space=api_pb2.FeasibleSpace(
- max=None, min=None, list=["3", "2", "6"])
- ),
- api_pb2.ParameterSpec(
- name="param-4",
- parameter_type=api_pb2.DOUBLE,
- feasible_space=api_pb2.FeasibleSpace(
- max="2.9", min="1", list=[], step="0.5")
- )
- ]
- ),
- max_trial_count=12,
- parallel_trial_count=3,
- )
-
- _, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
- self.assertEqual(code, grpc.StatusCode.OK)
-
- # Invalid cases.
- # Empty step.
- experiment_spec = api_pb2.ExperimentSpec(
- algorithm=api_pb2.AlgorithmSpec(
- algorithm_name="grid",
- ),
- parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
- parameters=[
- api_pb2.ParameterSpec(
- name="param-1",
- parameter_type=api_pb2.DOUBLE,
- feasible_space=api_pb2.FeasibleSpace(
- max="3", min="1", list=[])
- )
- ]
- ),
- )
-
- _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
- self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
- self.assertEqual(details, 'Param: param-1 step is nil')
-
- # Max trial count > search space combinations.
- experiment_spec = api_pb2.ExperimentSpec(
- algorithm=api_pb2.AlgorithmSpec(
- algorithm_name="grid",
- ),
- parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
- parameters=[
- api_pb2.ParameterSpec(
- name="param-1",
- parameter_type=api_pb2.INT,
- feasible_space=api_pb2.FeasibleSpace(
- max="2", min="1", list=[]),
- ),
- api_pb2.ParameterSpec(
- name="param-2",
- parameter_type=api_pb2.CATEGORICAL,
- feasible_space=api_pb2.FeasibleSpace(
- max=None, min=None, list=["cat1", "cat2"])
- ),
- api_pb2.ParameterSpec(
- name="param-4",
- parameter_type=api_pb2.DOUBLE,
- feasible_space=api_pb2.FeasibleSpace(
- max="2", min="1", list=[], step="0.5")
- )
- ]
- ),
- max_trial_count=15,
- )
-
- _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
- self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
- self.assertEqual(details, 'Max Trial Count: 15 > all possible search space combinations: 12')
-
-
-@pytest.fixture(scope='function', autouse=True)
-def tear_down():
- yield
- working_dir = os.getcwd()
- db_file = ["my_db.db", "my_db.db?check_same_thread=False.lock", "my_db.db-shm", "my_db.db-wal"]
- for fname in db_file:
- target_path = os.path.join(working_dir, fname)
- if os.path.isfile(target_path):
- os.remove(target_path)
-
-
-if __name__ == '__main__':
- unittest.main()
|