Fail loudly on unsupported Optuna parameter distributions (#2666)

The Optuna suggestion service mapped only UNIFORM and LOG_UNIFORM for int
and double parameters. A parameter declaring NORMAL or LOG_NORMAL fell
through every branch in _get_optuna_search_space and was left out of the
search space, so Optuna never sampled it. The hyperparameter was silently
dropped from tuning with no error or log.

Optuna has no native normal or log-normal distribution, so the backend
cannot honor those values. Add an explicit branch for int and double that
raises a ValueError naming the parameter and distribution, turning a
silent correctness bug into an actionable error. The hyperopt suggestion
service already supports these distributions, so the proto values are
valid input that this backend must reject explicitly rather than ignore.

Adds a parametrized regression test covering NORMAL and LOG_NORMAL for
both int and double parameters.

Fixes #2665

Signed-off-by: Saivedant Hava <[email protected]>

pkg/suggestion/v1beta1/optuna/base_service.py

@@ -129,6 +129,13 @@ class BaseOptunaService(object):
                         log=True,
                         step=1,
                     )
+                else:
+                    raise ValueError(
+                        f"Unsupported distribution "
+                        f"{api_pb2.Distribution.Name(param.distribution)} for parameter "
+                        f"{param.name}. The Optuna suggestion service supports only UNIFORM "
+                        f"and LOG_UNIFORM for int parameters."
+                    )
 
             elif param.type == DOUBLE:
                 if param.distribution in [api_pb2.UNIFORM, None]:
@@ -148,6 +155,13 @@ class BaseOptunaService(object):
                         log=True,
                         step=None,
                     )
+                else:
+                    raise ValueError(
+                        f"Unsupported distribution "
+                        f"{api_pb2.Distribution.Name(param.distribution)} for parameter "
+                        f"{param.name}. The Optuna suggestion service supports only UNIFORM "
+                        f"and LOG_UNIFORM for double parameters."
+                    )
 
             elif param.type in [CATEGORICAL, DISCRETE]:
                 # Categorical & Discrete parameters use a categorical distribution.

test/unit/v1beta1/suggestion/test_optuna_service.py

@@ -18,6 +18,12 @@ import pytest
 import utils
 
 from pkg.apis.manager.v1beta1.python import api_pb2
+from pkg.suggestion.v1beta1.internal import constant
+from pkg.suggestion.v1beta1.internal.search_space import (
+    HyperParameter,
+    HyperParameterSearchSpace,
+)
+from pkg.suggestion.v1beta1.optuna.base_service import BaseOptunaService
 from pkg.suggestion.v1beta1.optuna.service import OptunaService
 
 
@@ -113,7 +119,10 @@ class TestOptuna:
                             name="param-7",
                             parameter_type=api_pb2.INT,
                             feasible_space=api_pb2.FeasibleSpace(
-                                max="5", min="1", step="2", distribution=api_pb2.LOG_UNIFORM
+                                max="5",
+                                min="1",
+                                step="2",
+                                distribution=api_pb2.LOG_UNIFORM,
                             ),
                         ),
                         api_pb2.ParameterSpec(
@@ -127,14 +136,20 @@ class TestOptuna:
                             name="param-9",
                             parameter_type=api_pb2.DOUBLE,
                             feasible_space=api_pb2.FeasibleSpace(
-                                max="11", min="1", step="2.5", distribution=api_pb2.UNIFORM
+                                max="11",
+                                min="1",
+                                step="2.5",
+                                distribution=api_pb2.UNIFORM,
                             ),
                         ),
                         api_pb2.ParameterSpec(
                             name="param-10",
                             parameter_type=api_pb2.DOUBLE,
                             feasible_space=api_pb2.FeasibleSpace(
-                                max="11", min="1", step="2.5", distribution=api_pb2.LOG_UNIFORM
+                                max="11",
+                                min="1",
+                                step="2.5",
+                                distribution=api_pb2.LOG_UNIFORM,
                             ),
                         ),
                         api_pb2.ParameterSpec(
@@ -552,5 +567,38 @@ class TestOptuna:
         assert code == result
 
 
+class TestOptunaUnsupportedDistribution:
+    @pytest.mark.parametrize(
+        ["param_type", "distribution"],
+        [
+            ["double", api_pb2.NORMAL],
+            ["double", api_pb2.LOG_NORMAL],
+            ["int", api_pb2.NORMAL],
+            ["int", api_pb2.LOG_NORMAL],
+        ],
+    )
+    def test_unsupported_distribution_raises(self, param_type, distribution):
+        # The Optuna suggestion service only maps UNIFORM and LOG_UNIFORM. A param
+        # with NORMAL or LOG_NORMAL used to be dropped from the search space without
+        # any error, so the hyperparameter was silently never tuned (#2665). It must
+        # now fail loudly instead.
+        search_space = HyperParameterSearchSpace()
+        search_space.goal = constant.MAX_GOAL
+        if param_type == "double":
+            param = HyperParameter.double("x", "0.1", "0.9", "", distribution)
+        else:
+            param = HyperParameter.int("x", "1", "9", "1", distribution)
+        search_space.params = [param]
+
+        service = BaseOptunaService(
+            algorithm_name="random",
+            algorithm_config={},
+            search_space=search_space,
+        )
+
+        with pytest.raises(ValueError, match="Unsupported distribution"):
+            service.get_suggestions([], 1)
+
+
 if __name__ == "__main__":
     pytest.main()