From fb0909df34a3af836fce2790d92ac490cf4055c6 Mon Sep 17 00:00:00 2001
From: Shaokun Zhang <svz5418@i4-l-qxw5138-01.ad.psu.edu>
Date: Sat, 1 Apr 2023 19:08:21 -0400
Subject: [PATCH 01/15] first draft

---
 flaml/tune/searcher/blendsearch.py   | 181 ++++++++++++++++++++++++---
 flaml/tune/searcher/flow2.py         |  10 +-
 flaml/tune/searcher/search_thread.py | 167 ++++++++++++++++++++----
 flaml/tune/tune.py                   |  27 +---
 4 files changed, 311 insertions(+), 74 deletions(-)

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index 92a50e2640..0622d8b02d 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -165,6 +165,9 @@ def __init__(
         self._metric, self._mode = metric, mode
         self._use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
         self.lexico_objectives = lexico_objectives
+        self._histories = None
+        self._f_best = None
+        self._f_worst = None
         init_config = low_cost_partial_config or {}
         if not init_config:
             logger.info(
@@ -275,6 +278,41 @@ def __init__(
         if space is not None:
             self._init_search()
 
+    def update_fbest(
+        self,
+    ):
+        obj_initial = self.lexico_objectives["metrics"][0]
+        feasible_index = np.array([*range(len(self._histories[obj_initial]))])
+        for k_metric in self.lexico_objectives["metrics"]:
+            k_values = np.array(self._histories[k_metric])
+            feasible_value = k_values.take(feasible_index)
+            self._f_best[k_metric] = np.min(feasible_value)
+            self._f_worst[k_metric] = np.max(feasible_value)
+            if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
+                tolerance_bound = (
+                    self._f_best[k_metric]
+                    + self.lexico_objectives["tolerances"][k_metric]
+                )
+            else:
+                assert (
+                    self.lexico_objectives["tolerances"][k_metric][-1] == "%"
+                ), "String tolerance of {} should use %% as the suffix".format(k_metric)
+                tolerance_bound = self._f_best[k_metric] * (
+                    1
+                    + 0.01
+                    * float(
+                        self.lexico_objectives["tolerances"][k_metric].replace("%", "")
+                    )
+                )
+            feasible_index_filter = np.where(
+                feasible_value
+                <= max(
+                    tolerance_bound,
+                    self.lexico_objectives["targets"][k_metric],
+                )
+            )[0]
+            feasible_index = feasible_index.take(feasible_index_filter)
+
     def set_search_properties(
         self,
         metric: Optional[str] = None,
@@ -327,7 +365,7 @@ def set_search_properties(
                 self._set_deadline()
                 if self._input_cost_attr == "auto" and self._time_budget_s:
                     self.cost_attr = self._ls.cost_attr = TIME_TOTAL_S
-            if "metric_target" in spec:
+            if "metric_target" in spec:  # works only in online_searcher
                 self._metric_target = spec.get("metric_target")
             num_samples = spec.get("num_samples")
             if num_samples is not None:
@@ -356,7 +394,10 @@ def _init_search(self):
         self._set_deadline()
         self._is_ls_ever_converged = False
         self._subspace = {}  # the subspace for each trial id
-        self._metric_target = np.inf * self._ls.metric_op
+        if self.lexico_objectives is None:
+            self._metric_target = np.inf * self._ls.metric_op
+        else:
+            self._metric_target = {k: np.inf * self._ls.metric_op for k_metric self.lexico_objectives["metrics"]}
         self._search_thread_pool = {
             # id: int -> thread: SearchThread
             0: SearchThread(self._ls.mode, self._gs, self.cost_attr, self._eps)
@@ -381,7 +422,7 @@ def _init_search(self):
         self._gs_admissible_min = self._ls_bound_min.copy()
         self._gs_admissible_max = self._ls_bound_max.copy()
 
-        if self._metric_constraints:
+        if self._metric_constraints:  # check: remove
             self._metric_constraint_satisfied = False
             self._metric_constraint_penalty = [
                 self.penalty for _ in self._metric_constraints
@@ -416,6 +457,36 @@ def restore(self, checkpoint_path: str):
         self._start_time = time.time()
         self._set_deadline()
 
+    def _get_lexico_bound(self, metric, mode):
+
+        k_target = (
+            self.lexico_objectives["targets"][metric]
+            if mode == "min"
+            else -self.lexico_objectives["targets"][metric]
+        )
+        if not isinstance(self.lexico_objectives["tolerances"][metric], str):
+            tolerance_bound = (
+                self.f_best[metric]
+                + self.lexico_objectives["tolerances"][metric]
+            )
+        else:
+            assert (
+                self.lexico_objectives["tolerances"][metric][-1] == "%"
+            ), "String tolerance of {} should use %% as the suffix".format(
+                metric
+            )
+            tolerance_bound = self.f_best[metric] * (
+                1
+                + 0.01
+                * float(
+                    self.lexico_objectives["tolerances"][metric].replace(
+                        "%", ""
+                    )
+                )
+            )
+        bound = max(tolerance_bound, k_target)
+        return bound
+
     @property
     def metric_target(self):
         return self._metric_target
@@ -424,12 +495,12 @@ def metric_target(self):
     def is_ls_ever_converged(self):
         return self._is_ls_ever_converged
 
-    def on_trial_complete(
+    def on_trial_complete(  # check
         self, trial_id: str, result: Optional[Dict] = None, error: bool = False
     ):
         """search thread updater and cleaner."""
         metric_constraint_satisfied = True
-        if result and not error and self._metric_constraints:
+        if result and not error and self._metric_constraints:  # remove check
             # account for metric constraints if any
             objective = result[self._metric]
             for i, constraint in enumerate(self._metric_constraints):
@@ -460,6 +531,12 @@ def on_trial_complete(
             )
             del self._trial_proposed_by[trial_id]
         if result:
+            if self.lexico_objectives:
+                if self._histories is None:
+                    self._histories, self._f_best, self._f_worst = defaultdict(list), {}, {}
+                for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
+                    self._histories[k_metric].append(result[k_metric] if k_mode == "min" else -result[k_metric])
+                self.update_fbest()
             config = result.get("config", {})
             if not config:
                 for key, value in result.items():
@@ -477,10 +554,28 @@ def on_trial_complete(
                 self._result[signature] = result
                 # update target metric if improved
                 objective = result[self._ls.metric]
-                if (objective - self._metric_target) * self._ls.metric_op < 0:
+                if self.lexico_objectives is None and (objective - self._metric_target) * self._ls.metric_op < 0:
                     self._metric_target = objective
-                    if self._ls.resource:
-                        self._best_resource = config[self._ls.resource_attr]
+                else:
+                    for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
+                        bound = _get_lexico_bound(k_metric, k_mode)
+                        if (objective[k_metric] * self._ls.metric_op < bound) and (self._metric_target[k_metric] * self._ls.metric_op < bound):
+                            continue
+                        elif objective[k_metric] * self._ls.metric_op < self._metric_target[k_metric] * self._ls.metric_op:
+                            self._metric_target = objective
+                            break
+                        else:
+                            break
+                    for k_metr in self.lexico_objectives["metrics"]:
+                        if objective[k_metr] == self._metric_target[k_metr]:
+                            continue
+                        elif objective[k_metr] * self._ls.metric_op < self._metric_target[k_metr] * self._ls.metric_op:
+                            self._metric_target = objective
+                            break
+                        else:
+                            break
+                if self._ls.resource:
+                    self._best_resource = config[self._ls.resource_attr]
                 if thread_id:
                     if not self._metric_constraint_satisfied:
                         # no point has been found to satisfy metric constraint
@@ -591,7 +686,7 @@ def _update_admissible_region(
                 elif value < admissible_min[key]:
                     admissible_min[key] = value
 
-    def _create_condition(self, result: Dict) -> bool:
+    def _create_condition(self, result: Dict) -> bool:  # check
         """create thread condition"""
         if len(self._search_thread_pool) < 2:
             return True
@@ -600,7 +695,7 @@ def _create_condition(self, result: Dict) -> bool:
         )
         return result[self._ls.metric] * self._ls.metric_op < obj_median
 
-    def _clean(self, thread_id: int):
+    def _clean(self, thread_id: int):  # check
         """delete thread and increase admissible region if converged,
         merge local threads if they are close
         """
@@ -643,7 +738,7 @@ def _clean(self, thread_id: int):
         if create_new:
             self._create_thread_from_best_candidate()
 
-    def _create_thread_from_best_candidate(self):
+    def _create_thread_from_best_candidate(self):  # check
         # find the best start point
         best_trial_id = None
         obj_best = None
@@ -683,7 +778,36 @@ def _expand_admissible_region(self, lower, upper, space):
                 upper[key] += self._ls.STEPSIZE
                 lower[key] -= self._ls.STEPSIZE
 
-    def _inferior(self, id1: int, id2: int) -> bool:
+    def _lexico_inferior(self, objective_1: Union[dict, float], objective_2: Union[dict, float]) -> bool:
+        if isinstance(objective_1, dict) and isinstance(objective_2, dict):
+            for k_metric, k_mode in zip(
+                self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
+            ):
+                bound = _get_lexico_bound(k_metric, k_mode):
+                if (objective_1[k_metric] < bound) and (self.objective_2[k_metric] < bound):
+                    continue
+                elif objective_1[k_metric] < self.objective_2[k_metric]:
+                    return True
+                else:
+                    return False
+            for k_metr in self.lexico_objectives["metrics"]:
+                if objective_1[k_metr] == self.objective_2[k_metr]:
+                    continue
+                elif objective_1[k_metr] < self.objective_2[k_metr]:
+                    return True
+                else:
+                    return False
+        else:
+            priority_1 = - objective_1[self.lexico_objectives[self.lexico_objectives["metrics"]
+                                                              [0]]] if isinstance(objective_1, dict) else objective_1
+            priority_2 = - objective_2[self.lexico_objectives[self.lexico_objectives["metrics"]
+                                                              [0]]] if isinstance(objective_2, dict) else objective_2
+            if priority_1 > priority_2:
+                return True
+            else:
+                return False
+
+    def _inferior(self, id1: int, id2: int) -> bool:  # check
         """whether thread id1 is inferior to id2"""
         t1 = self._search_thread_pool[id1]
         t2 = self._search_thread_pool[id2]
@@ -704,6 +828,12 @@ def on_trial_result(self, trial_id: str, result: Dict):
             return
         if result and self._metric_constraints:
             result[self._metric + self.lagrange] = result[self._metric]
+            if self.lexico_objectives:
+                if self._histories is None:
+                    self._histories, self._f_best, self._f_worst = defaultdict(list), {}, {}
+                for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
+                    self._histories[k_metric].append(result[k_metric] if k_mode == "min" else -result[k_metric])
+                self.update_fbest()
         self._search_thread_pool[thread_id].on_trial_result(trial_id, result)
 
     def suggest(self, trial_id: str) -> Optional[Dict]:
@@ -914,26 +1044,37 @@ def _select_thread(self) -> Tuple:
             num_proposed = num_finished + len(self._trial_proposed_by)
             min_eci = max(self._num_samples - num_proposed, 0)
         # update priority
-        max_speed = 0
-        for thread in self._search_thread_pool.values():
-            if thread.speed > max_speed:
-                max_speed = thread.speed
+        if self.lexico_objectives is not None:
+            max_speed = 0
+            for thread in self._search_thread_pool.values():
+                if thread.speed > max_speed:
+                    max_speed = thread.speed
+        else:
+            max_speed = {k: 0, for k in self.lexico_objectives["metrics"]}
+            for k_metric in self.lexico_objectives["metrics"]:
+                for thread in self._search_thread_pool.values():
+                    if thread.speed[k_metric] > max_speed[k_metric]:
+                        max_speed[k_metric] = thread.speed[k_metric]
         for thread in self._search_thread_pool.values():
             thread.update_eci(self._metric_target, max_speed)
             if thread.eci < min_eci:
                 min_eci = thread.eci
         for thread in self._search_thread_pool.values():
             thread.update_priority(min_eci)
-
         top_thread_id = backup_thread_id = 0
+        # how to compare global search priority with local search
         priority1 = priority2 = self._search_thread_pool[0].priority
         for thread_id, thread in self._search_thread_pool.items():
             if thread_id and thread.can_suggest:
                 priority = thread.priority
-                if priority > priority1:
+                inferior_condition1 = (self.lexico_objectives and _lexico_inferior(priority, priority1)) or (
+                    self.lexico_objectives is None and priority > priority1)
+                inferior_condition2 = (self.lexico_objectives and _lexico_inferior(priority, priority2)) or (
+                    self.lexico_objectives is None and priority > priority2)
+                if inferior_condition1:
                     priority1 = priority
                     top_thread_id = thread_id
-                if priority > priority2 or backup_thread_id == 0:
+                if inferior_condition2 or backup_thread_id == 0:
                     priority2 = priority
                     backup_thread_id = thread_id
         return top_thread_id, backup_thread_id
@@ -1183,4 +1324,4 @@ def on_trial_complete(
         return
 
     def on_trial_result(self, trial_id: str, result: Dict):
-        return
+        return
\ No newline at end of file
diff --git a/flaml/tune/searcher/flow2.py b/flaml/tune/searcher/flow2.py
index 035e3d8688..97c5ad1d5e 100644
--- a/flaml/tune/searcher/flow2.py
+++ b/flaml/tune/searcher/flow2.py
@@ -53,7 +53,6 @@ def __init__(
         lexico_objectives=None,
     ):
         """Constructor.
-
         Args:
             init_config: a dictionary of a partial or full initial config,
                 e.g., from a subset of controlled dimensions
@@ -143,6 +142,7 @@ def __init__(
         self.max_resource = max_resource
         self._resource = None
         self._f_best = None  # only use for lexico_comapre. It represent the best value achieved by lexico_flow.
+        self.op_dimension = None 
         self._step_lb = np.Inf
         self._histories = None  # only use for lexico_comapre. It records the result of historical configurations.
         if space is not None:
@@ -200,6 +200,7 @@ def _init_search(self):
         self.incumbent = {}
         self.incumbent = self.normalize(self.best_config)  # flattened
         self.best_obj = self.cost_incumbent = None
+        self.pre_best_obj = None
         self.dim = len(self._tunable_keys)  # total # tunable dimensions
         self._direction_tried = None
         self._num_complete4incumbent = self._cost_complete4incumbent = 0
@@ -283,7 +284,6 @@ def complete_config(
         upper: Optional[Dict] = None,
     ) -> Tuple[Dict, Dict]:
         """Generate a complete config from the partial config input.
-
         Add minimal resource to config if available.
         """
         disturb = self._reset_times and partial_config == self.init_config
@@ -446,6 +446,7 @@ def lexico_compare(self, result) -> bool:
                 ):
                     continue
                 elif result[k_metric] < self.best_obj[k_metric]:
+                    self.op_dimension = k_metric
                     return True
                 else:
                     return False
@@ -453,6 +454,7 @@ def lexico_compare(self, result) -> bool:
                 if result[k_metr] == self.best_obj[k_metr]:
                     continue
                 elif result[k_metr] < self.best_obj[k_metr]:
+                    self.op_dimension = k_metric
                     return True
                 else:
                     return False
@@ -489,6 +491,7 @@ def on_trial_complete(
                     or (self.lexico_objectives is None and obj < self.best_obj)
                     or (self.lexico_objectives is not None and self.lexico_compare(obj))
                 ):
+                    self.pre_best_obj = self.best_obj
                     self.best_obj = obj
                     self.best_config, self.step = self._configs[trial_id]
                     self.incumbent = self.normalize(self.best_config)
@@ -555,6 +558,7 @@ def on_trial_result(self, trial_id: str, result: Dict):
                     or (self.lexico_objectives is None and obj < self.best_obj)
                     or (self.lexico_objectives is not None and self.lexico_compare(obj))
                 ):
+                    self.pre_best_obj = self.best_obj
                     self.best_obj = obj
                     config = self._configs[trial_id][0]
                     if self.best_config != config:
@@ -752,4 +756,4 @@ def reach(self, other: Searcher) -> bool:
                 for key in self._tunable_keys
             ]
         )
-        return np.linalg.norm(delta) <= self.step
+        return np.linalg.norm(delta) <= self.step
\ No newline at end of file
diff --git a/flaml/tune/searcher/search_thread.py b/flaml/tune/searcher/search_thread.py
index 9bb58a8ea3..0c507d62c2 100644
--- a/flaml/tune/searcher/search_thread.py
+++ b/flaml/tune/searcher/search_thread.py
@@ -43,13 +43,26 @@ def __init__(
         )
         self._eps = eps
         self.cost_best2 = 0
-        self.obj_best1 = self.obj_best2 = getattr(
-            search_alg, "best_obj", np.inf
-        )  # inherently minimize
+        self.lexico_objectives = getattr(self._search_alg, "lexico_objectives", None)
+
+        # get obj 1 and obj 2
+        if self._is_ls and self.lexico_objectives:
+            self.obj_best1 = self.obj_best2 = {}
+            for k in self.lexico_objectives["metrics"]:
+                self.obj_best1[k] = self.obj_best2[k] = np.inf if getattr(
+                    search_alg, "best_obj", None) is None else search_alg.best_obj[k]
+        else:
+            self.obj_best1 = self.obj_best2 = getattr(
+                search_alg, "best_obj", np.inf
+            )  # inherently minimize
+
         self.best_result = None
         # eci: estimated cost for improvement
         self.eci = self.cost_best
-        self.priority = self.speed = 0
+        if self._is_ls and self.lexico_objectives:
+            self.priority = self.speed = {}
+        else:
+            self.priority = self.speed = 0
         self._init_config = True
         self.running = 0  # the number of running trials from the thread
         self.cost_attr = cost_attr
@@ -85,32 +98,127 @@ def suggest(self, trial_id: str) -> Optional[Dict]:
             self.running += 1
         return config
 
+    def _get_lexico_bound(self, metric, mode):
+
+        k_target = (
+            self.lexico_objectives["targets"][metric]
+            if mode == "min"
+            else -self.lexico_objectives["targets"][metric]
+        )
+        if not isinstance(self.lexico_objectives["tolerances"][metric], str):
+            tolerance_bound = (
+                self.f_best[metric]
+                + self.lexico_objectives["tolerances"][metric]
+            )
+        else:
+            assert (
+                self.lexico_objectives["tolerances"][metric][-1] == "%"
+            ), "String tolerance of {} should use %% as the suffix".format(
+                metric
+            )
+            tolerance_bound = self.f_best[metric] * (
+                1
+                + 0.01
+                * float(
+                    self.lexico_objectives["tolerances"][metric].replace(
+                        "%", ""
+                    )
+                )
+            )
+        bound = max(tolerance_bound, k_target)
+        return bound
+
     def update_priority(self, eci: Optional[float] = 0):
-        # optimistic projection
-        self.priority = eci * self.speed - self.obj_best1
+        if self.lexico_objectives and self._is_ls:
+            feasible_flag = True
+            for k_metric, k_mode in zip(
+                self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
+            ):
+                k_bound = _get_lexico_bound(k_metric, k_mode)
+                if self.obj_best1[k_metric] < k_bound:
+                    self.priority[k_metric] = self.obj_best1[k_metric]
+                    continue
+                elif self.obj_best1[k_metric] < self.f_best[k_metric]:
+                    raise ValueError(
+                        "Bug exists: Performance of a specific search thread is not possible to be better than global best.")
+                else:
+                    start_point = self.obj_best1[k_metric] if feasible_flag else self.f_worst[k_metric]
+                    if start_point <= k_bound:
+                        continue
+                    elif start_point > k_bound and start_point - eci * self.speed[k_metric] < k_bound:
+                        eci -= (start_point - k_bound) / self.speed[k_metric]
+                        self.priority[k_metric] = k_bound
+                    elif eci == 0:
+                        self.priority[k_metric] = self.f_worst[k_metric]
+                    else:
+                        self.priority[k_metric] = start_point - eci * self.speed[k_metric] 
+                        eci = 0
+                    feasible_flag = False
+        else:
+            self.priority = eci * self.speed - self.obj_best1
 
-    def update_eci(self, metric_target: float, max_speed: Optional[float] = np.inf):
+    def update_eci(self, metric_target: Union[float, dict], max_speed: Optional[Union[float, dict]], f_best: Optional[dict], f_worst: Optional[dict]):
         # calculate eci: estimated cost for improvement over metric_target
-        best_obj = metric_target * self._metric_op
-        if not self.speed:
-            self.speed = max_speed
-        self.eci = max(
-            self.cost_total - self.cost_best1, self.cost_best1 - self.cost_best2
-        )
-        if self.obj_best1 > best_obj and self.speed > 0:
-            self.eci = max(self.eci, 2 * (self.obj_best1 - best_obj) / self.speed)
+        if self.lexico_objectives is None or not self._is_ls:  
+            best_obj = metric_target * self._metric_op
+            if not self.speed:
+                self.speed = max_speed
+            self.eci = max(
+                self.cost_total - self.cost_best1, self.cost_best1 - self.cost_best2
+            )
+            if self.obj_best1 > best_obj and self.speed > 0:
+                self.eci = max(self.eci, 2 * (self.obj_best1 - best_obj) / self.speed)
+        else:  # Optimization lexicographically
+            self.eci = max(
+                self.cost_total - self.cost_best1, self.cost_best1 - self.cost_best2
+            )
+            eci_last = 0
+            feasible_flag = True
+            # TODO: finish 3/31
+            for k_metric, k_mode in zip(
+                self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
+            ):
+                if not self.speed[k_metric]:
+                    self.speed[k_metric] = max_speed
+                k_bound = _get_lexico_bound(k_metric, k_mode)
+                if self.obj_best1[k_metric] < k_bound:
+                    continue
+                elif self.obj_best1[k_metric] < self.f_best[k_metric]:
+                    raise ValueError(
+                        "Bug exists: Performance of a specific search thread is not possible to be better than global best.")
+                else:               
+                    if self.speed[k_metric] > 0: # check again in which cases speed[k_metric] <= 0
+                        start_point = self.obj_best1[k_metric] if feasible_flag else self.f_worst[k_metric]
+                        eci_last + = (start_point - k_bound) / self.speed[k_metric]
+                    feasible_flag = False
+            self.eci = max(self.eci, 2 * eci_last)
 
     def _update_speed(self):
         # calculate speed; use 0 for invalid speed temporarily
-        if self.obj_best2 > self.obj_best1:
+        if self.lexico_objectives is None and self.obj_best2 > self.obj_best1:
             # discount the speed if there are unfinished trials
             self.speed = (
                 (self.obj_best2 - self.obj_best1)
                 / self.running
                 / (max(self.cost_total - self.cost_best2, self._eps))
             )
+        elif self.lexico_objectives != None and self.obj_best2 != self.obj_best1:
+            op_dimension = self._search_alg.op_dimension
+            op_index = self.lexico_objectives["metrics"].index(op_dimension)
+            self.speed[op_dimension] = (
+                (self.obj_best2[op_dimension] - self.obj_best1[op_dimension])
+                / self.running
+                / (max(self.cost_total - self.cost_best2, self._eps))
+            )
+            for i in range(0, len(metrics)):
+                if i != op_index:
+                    self.speed[self.lexico_objectives["metrics"][i]] = 0
         else:
-            self.speed = 0
+            if self.lexico_objectives is None:
+                self.speed = 0
+            else:
+                for i in range(0, len(metrics)):
+                    self.speed[self.lexico_objectives["metrics"][i]] = 0
 
     def on_trial_complete(
         self, trial_id: str, result: Optional[Dict] = None, error: bool = False
@@ -138,20 +246,25 @@ def on_trial_complete(
         if result:
             self.cost_last = result.get(self.cost_attr, 1)
             self.cost_total += self.cost_last
-            if self._search_alg.metric in result and (
-                getattr(self._search_alg, "lexico_objectives", None) is None
-            ):
-                # TODO: Improve this behavior. When lexico_objectives is provided to CFO,
-                # related variables are not callable.
-                obj = result[self._search_alg.metric] * self._metric_op
-                if obj < self.obj_best1 or self.best_result is None:
+            if self._search_alg.metric in result:
+                if self.lexico_objectives is None:
+                    obj = result[self._search_alg.metric] * self._metric_op
+                else:
+                    obj = {}
+                    for k, m in zip(self._search_alg.lexico_objectives["metrics"], self._search_alg.lexico_objectives["modes"]):
+                        obj[k] = -result[k] if m == "max" else result[k]
+                if self.best_result is None or (self.lexico_objectives is None and obj < self.obj_best1) or (self.lexico_objectives is not None and obj == self._search_alg.best_obj):
                     self.cost_best2 = self.cost_best1
                     self.cost_best1 = self.cost_total
-                    self.obj_best2 = obj if np.isinf(self.obj_best1) else self.obj_best1
+                    if self.lexico_objectives is None:
+                        self.obj_best2 = obj if np.isinf(self.obj_best1) else self.obj_best1
+                    else:
+                        self.obj_best2 = obj if np.isinf(
+                            self.obj_best1[self.lexico_objectives["metrics"][0]]) else self.obj_best1
                     self.obj_best1 = obj
                     self.cost_best = self.cost_last
                     self.best_result = result
-            if getattr(self._search_alg, "lexico_objectives", None) is None:
+            if self.lexico_objectives is None:
                 # TODO: Improve this behavior. When lexico_objectives is provided to CFO,
                 # related variables are not callable.
                 self._update_speed()
@@ -191,4 +304,4 @@ def reach(self, thread) -> bool:
     @property
     def can_suggest(self) -> bool:
         """Whether the thread can suggest new configs."""
-        return self._search_alg.can_suggest
+        return self._search_alg.can_suggest
\ No newline at end of file
diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py
index e00ae1143c..9bcd860917 100644
--- a/flaml/tune/tune.py
+++ b/flaml/tune/tune.py
@@ -145,19 +145,15 @@ def best_result(self) -> Dict:
 def report(_metric=None, **kwargs):
     """A function called by the HPO application to report final or intermediate
     results.
-
     Example:
-
     ```python
     import time
     from flaml import tune
-
     def compute_with_config(config):
         current_time = time.time()
         metric2minimize = (round(config['x'])-95000)**2
         time2eval = time.time() - current_time
         tune.report(metric2minimize=metric2minimize, time2eval=time2eval)
-
     analysis = tune.run(
         compute_with_config,
         config={
@@ -166,15 +162,12 @@ def compute_with_config(config):
         },
         metric='metric2minimize', mode='min',
         num_samples=1000000, time_budget_s=60, use_ray=False)
-
     print(analysis.trials[-1].last_result)
     ```
-
     Args:
         _metric: Optional default anonymous metric for ``tune.report(value)``.
             (For compatibility with ray.tune.report)
         **kwargs: Any key value pair to be reported.
-
     Raises:
         StopIteration (when not using ray, i.e., _use_ray=False):
             A StopIteration exception is raised if the trial has been signaled to stop.
@@ -251,13 +244,10 @@ def run(
     **ray_args,
 ):
     """The trigger for HPO.
-
     Example:
-
     ```python
     import time
     from flaml import tune
-
     def compute_with_config(config):
         current_time = time.time()
         metric2minimize = (round(config['x'])-95000)**2
@@ -269,7 +259,6 @@ def compute_with_config(config):
         # if the failure indicates a config is bad,
         # report a bad metric value like np.inf or -np.inf
         # depending on metric mode being min or max
-
     analysis = tune.run(
         compute_with_config,
         config={
@@ -278,10 +267,8 @@ def compute_with_config(config):
         },
         metric='metric2minimize', mode='min',
         num_samples=-1, time_budget_s=60, use_ray=False)
-
     print(analysis.trials[-1].last_result)
     ```
-
     Args:
         evaluation_function: A user-defined evaluation function.
             It takes a configuration as input, outputs a evaluation
@@ -293,7 +280,6 @@ def compute_with_config(config):
         low_cost_partial_config: A dictionary from a subset of
             controlled dimensions to the initial low-cost values.
             e.g., ```{'n_estimators': 4, 'max_leaves': 4}```
-
         cat_hp_cost: A dictionary from a subset of categorical dimensions
             to the relative cost of each choice.
             e.g., ```{'tree_method': [1, 1, 2]}```
@@ -312,7 +298,6 @@ def compute_with_config(config):
             needing to re-compute the trial. Must be the same or shorter length than
             points_to_evaluate.
             e.g.,
-
     ```python
     points_to_evaluate = [
         {"b": .99, "cost_related": {"a": 3}},
@@ -320,10 +305,8 @@ def compute_with_config(config):
     ]
     evaluated_rewards = [3.0]
     ```
-
             means that you know the reward for the first config in
             points_to_evaluate is 3.0 and want to inform run().
-
         resource_attr: A string to specify the resource dimension used by
             the scheduler via "scheduler".
         min_resource: A float of the minimal resource to use for the resource_attr.
@@ -366,7 +349,6 @@ def easy_objective(config):
         search_alg: An instance of BlendSearch as the search algorithm
             to be used. The same instance can be used for iterative tuning.
             e.g.,
-
     ```python
     from flaml import BlendSearch
     algo = BlendSearch(metric='val_loss', mode='min',
@@ -377,7 +359,6 @@ def easy_objective(config):
             search_alg=algo, use_ray=False)
         print(analysis.trials[-1].last_result)
     ```
-
         verbose: 0, 1, 2, or 3. If ray or spark backend is used, their verbosity will be
             affected by this argument. 0 = silent, 1 = only status updates,
             2 = status and brief trial results, 3 = status and detailed trial results.
@@ -391,7 +372,6 @@ def easy_objective(config):
             [parallel tuning](../../Use-Cases/Tune-User-Defined-Function#parallel-tuning).
         config_constraints: A list of config constraints to be satisfied.
             e.g., ```config_constraints = [(mem_size, '<=', 1024**3)]```
-
             mem_size is a function which produces a float number for the bytes
             needed for a config.
             It is used to skip configs which do not fit in memory.
@@ -498,7 +478,7 @@ def easy_objective(config):
 
     if lexico_objectives is not None:
         logger.warning(
-            "If lexico_objectives is not None, search_alg is forced to be CFO"
+            "If lexico_objectives is not None, search_alg is forced to be CFO or Blendsearch" 
         )
         search_alg = None
     if search_alg is None:
@@ -530,7 +510,7 @@ def easy_objective(config):
                 )
             metric = metric or DEFAULT_METRIC
         else:
-            SearchAlgorithm = CFO
+            SearchAlgorithm = CFO if lexico_objectives["lexico_algorithm"] == "CFO" else BlendSearch
             logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
             metric = lexico_objectives["metrics"][0] or DEFAULT_METRIC
         search_alg = SearchAlgorithm(
@@ -668,11 +648,9 @@ def easy_objective(config):
         launch one trial per executor. However, sometimes we can launch more trials than
         the number of executors (e.g., local mode). In this case, we can set the environment
         variable `FLAML_MAX_CONCURRENT` to override the detected `num_executors`.
-
         `max_concurrent` is the maximum number of concurrent trials defined by `search_alg`,
         `FLAML_MAX_CONCURRENT` will also be used to override `max_concurrent` if `search_alg`
         is not an instance of `ConcurrencyLimiter`.
-
         The final number of concurrent trials is the minimum of `max_concurrent` and
         `num_executors`.
         """
@@ -851,3 +829,4 @@ def easy_objective(config):
             _runner = old_runner
             logger.handlers = old_handlers
             logger.setLevel(old_level)
+            
\ No newline at end of file

From ac258adb43fbacc55fc763c3ddc42b48f016ac81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Tue, 13 Jun 2023 21:43:15 +0800
Subject: [PATCH 02/15] update

---
 flaml/tune/searcher/blendsearch.py   |  86 +++++++++-------------
 flaml/tune/searcher/search_thread.py | 104 ++++++++++-----------------
 flaml/tune/tune.py                   |   2 +-
 3 files changed, 73 insertions(+), 119 deletions(-)

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index 0128f221a8..b44bedf264 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -3,6 +3,7 @@
 #  * Licensed under the MIT License. See LICENSE file in the
 #  * project root for license information.
 from typing import Dict, Optional, List, Tuple, Callable, Union
+from collections import defaultdict
 import numpy as np
 import time
 import pickle
@@ -165,7 +166,6 @@ def __init__(
         self.lexico_objectives = lexico_objectives
         self._histories = None
         self._f_best = None
-        self._f_worst = None
         init_config = low_cost_partial_config or {}
         if not init_config:
             logger.info(
@@ -277,7 +277,6 @@ def update_fbest(
             k_values = np.array(self._histories[k_metric])
             feasible_value = k_values.take(feasible_index)
             self._f_best[k_metric] = np.min(feasible_value)
-            self._f_worst[k_metric] = np.max(feasible_value)
             if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
                 tolerance_bound = (
                     self._f_best[k_metric]
@@ -386,8 +385,6 @@ def _init_search(self):
         self._subspace = {}  # the subspace for each trial id
         if self.lexico_objectives is None:
             self._metric_target = np.inf * self._ls.metric_op
-        else:
-            self._metric_target = {k: np.inf * self._ls.metric_op for k_metric self.lexico_objectives["metrics"]}
         self._search_thread_pool = {
             # id: int -> thread: SearchThread
             0: SearchThread(self._ls.mode, self._gs, self.cost_attr, self._eps)
@@ -412,7 +409,7 @@ def _init_search(self):
         self._gs_admissible_min = self._ls_bound_min.copy()
         self._gs_admissible_max = self._ls_bound_max.copy()
 
-        if self._metric_constraints:  # check: remove
+        if self._metric_constraints:  
             self._metric_constraint_satisfied = False
             self._metric_constraint_penalty = [self.penalty for _ in self._metric_constraints]
         else:
@@ -454,7 +451,7 @@ def _get_lexico_bound(self, metric, mode):
         )
         if not isinstance(self.lexico_objectives["tolerances"][metric], str):
             tolerance_bound = (
-                self.f_best[metric]
+                self._f_best[metric]
                 + self.lexico_objectives["tolerances"][metric]
             )
         else:
@@ -515,7 +512,7 @@ def on_trial_complete(  # check
         if result:
             if self.lexico_objectives:
                 if self._histories is None:
-                    self._histories, self._f_best, self._f_worst = defaultdict(list), {}, {}
+                    self._histories, self._f_best = defaultdict(list), {}
                 for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
                     self._histories[k_metric].append(result[k_metric] if k_mode == "min" else -result[k_metric])
                 self.update_fbest()
@@ -533,27 +530,10 @@ def on_trial_complete(  # check
                 self._cost_used += result.get(self.cost_attr, 0)
                 self._result[signature] = result
                 # update target metric if improved
-                objective = result[self._ls.metric]
-                if self.lexico_objectives is None and (objective - self._metric_target) * self._ls.metric_op < 0:
-                    self._metric_target = objective
-                else:
-                    for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                        bound = _get_lexico_bound(k_metric, k_mode)
-                        if (objective[k_metric] * self._ls.metric_op < bound) and (self._metric_target[k_metric] * self._ls.metric_op < bound):
-                            continue
-                        elif objective[k_metric] * self._ls.metric_op < self._metric_target[k_metric] * self._ls.metric_op:
-                            self._metric_target = objective
-                            break
-                        else:
-                            break
-                    for k_metr in self.lexico_objectives["metrics"]:
-                        if objective[k_metr] == self._metric_target[k_metr]:
-                            continue
-                        elif objective[k_metr] * self._ls.metric_op < self._metric_target[k_metr] * self._ls.metric_op:
-                            self._metric_target = objective
-                            break
-                        else:
-                            break
+                if self.lexico_objectives is None:
+                    objective = result[self._ls.metric]
+                    if (objective - self._metric_target) * self._ls.metric_op < 0:
+                        self._metric_target = objective
                 if self._ls.resource:
                     self._best_resource = config[self._ls.resource_attr]
                 if thread_id:
@@ -740,35 +720,31 @@ def _expand_admissible_region(self, lower, upper, space):
                 upper[key] += self._ls.STEPSIZE
                 lower[key] -= self._ls.STEPSIZE
 
-    def _lexico_inferior(self, objective_1: Union[dict, float], objective_2: Union[dict, float]) -> bool:
-        if isinstance(objective_1, dict) and isinstance(objective_2, dict):
+    def _priority_inferior(self, priority_1: Union[dict, float], priority_2: Union[dict, float]) -> bool:
+        if self.lexico_objectives:
             for k_metric, k_mode in zip(
                 self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
             ):
-                bound = _get_lexico_bound(k_metric, k_mode):
-                if (objective_1[k_metric] < bound) and (self.objective_2[k_metric] < bound):
+                bound = self._get_lexico_bound(k_metric, k_mode)
+                if (priority_1[k_metric] < bound) and (priority_2[k_metric] < bound):
                     continue
-                elif objective_1[k_metric] < self.objective_2[k_metric]:
+                elif priority_1[k_metric] < priority_2[k_metric]:
                     return True
                 else:
                     return False
             for k_metr in self.lexico_objectives["metrics"]:
-                if objective_1[k_metr] == self.objective_2[k_metr]:
+                if priority_1[k_metr] == priority_2[k_metr]:
                     continue
-                elif objective_1[k_metr] < self.objective_2[k_metr]:
+                elif priority_1[k_metr] < priority_2[k_metr]:
                     return True
                 else:
                     return False
         else:
-            priority_1 = - objective_1[self.lexico_objectives[self.lexico_objectives["metrics"]
-                                                              [0]]] if isinstance(objective_1, dict) else objective_1
-            priority_2 = - objective_2[self.lexico_objectives[self.lexico_objectives["metrics"]
-                                                              [0]]] if isinstance(objective_2, dict) else objective_2
-            if priority_1 > priority_2:
+            if priority_1 > priority_2: 
                 return True
             else:
                 return False
-
+            
     def _inferior(self, id1: int, id2: int) -> bool:  # check
         """whether thread id1 is inferior to id2"""
         t1 = self._search_thread_pool[id1]
@@ -792,7 +768,7 @@ def on_trial_result(self, trial_id: str, result: Dict):
             result[self._metric + self.lagrange] = result[self._metric]
             if self.lexico_objectives:
                 if self._histories is None:
-                    self._histories, self._f_best, self._f_worst = defaultdict(list), {}, {}
+                    self._histories, self._f_best = defaultdict(list), {}
                 for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
                     self._histories[k_metric].append(result[k_metric] if k_mode == "min" else -result[k_metric])
                 self.update_fbest()
@@ -998,19 +974,31 @@ def _select_thread(self) -> Tuple:
             num_proposed = num_finished + len(self._trial_proposed_by)
             min_eci = max(self._num_samples - num_proposed, 0)
         # update priority
-        if self.lexico_objectives is not None:
+        if self.lexico_objectives is None:
             max_speed = 0
+            min_speed = float("inf")
             for thread in self._search_thread_pool.values():
                 if thread.speed > max_speed:
                     max_speed = thread.speed
+                if thread.speed < min_speed:
+                    min_speed = thread.speed
         else:
-            max_speed = {k: 0, for k in self.lexico_objectives["metrics"]}
+            max_speed = {k: 0 for k in self.lexico_objectives["metrics"]}
+            min_speed = {k: float("inf") for k in self.lexico_objectives["metrics"]}
             for k_metric in self.lexico_objectives["metrics"]:
                 for thread in self._search_thread_pool.values():
                     if thread.speed[k_metric] > max_speed[k_metric]:
                         max_speed[k_metric] = thread.speed[k_metric]
+                    if thread.speed[k_metric] < min_speed[k_metric]:
+                        min_speed[k_metric] = thread.speed[k_metric]
         for thread in self._search_thread_pool.values():
-            thread.update_eci(self._metric_target, max_speed)
+            if self.lexico_objectives is None:
+                thread.update_eci(self._metric_target, max_speed, min_speed)
+            else: 
+                _metric_1st = self.lexico_objectives["metrics"][0]
+                _op_1st = self.lexico_objectives["modes"][_metric_1st]
+                _lexico_target = self._f_best[_metric_1st] if _op_1st == "min" else -1 * self._f_best[_metric_1st]
+                thread.update_eci(_lexico_target, max_speed, min_speed)
             if thread.eci < min_eci:
                 min_eci = thread.eci
         for thread in self._search_thread_pool.values():
@@ -1021,14 +1009,10 @@ def _select_thread(self) -> Tuple:
         for thread_id, thread in self._search_thread_pool.items():
             if thread_id and thread.can_suggest:
                 priority = thread.priority
-                inferior_condition1 = (self.lexico_objectives and _lexico_inferior(priority, priority1)) or (
-                    self.lexico_objectives is None and priority > priority1)
-                inferior_condition2 = (self.lexico_objectives and _lexico_inferior(priority, priority2)) or (
-                    self.lexico_objectives is None and priority > priority2)
-                if inferior_condition1:
+                if self._priority_inferior(priority, priority1):
                     priority1 = priority
                     top_thread_id = thread_id
-                if inferior_condition2 or backup_thread_id == 0:
+                if self._priority_inferior(priority, priority2) or backup_thread_id == 0:
                     priority2 = priority
                     backup_thread_id = thread_id
         return top_thread_id, backup_thread_id
diff --git a/flaml/tune/searcher/search_thread.py b/flaml/tune/searcher/search_thread.py
index 7b8320f81a..3cadac6a89 100644
--- a/flaml/tune/searcher/search_thread.py
+++ b/flaml/tune/searcher/search_thread.py
@@ -98,7 +98,7 @@ def _get_lexico_bound(self, metric, mode):
         )
         if not isinstance(self.lexico_objectives["tolerances"][metric], str):
             tolerance_bound = (
-                self.f_best[metric]
+                self._f_best[metric]
                 + self.lexico_objectives["tolerances"][metric]
             )
         else:
@@ -107,7 +107,7 @@ def _get_lexico_bound(self, metric, mode):
             ), "String tolerance of {} should use %% as the suffix".format(
                 metric
             )
-            tolerance_bound = self.f_best[metric] * (
+            tolerance_bound = self._f_best[metric] * (
                 1
                 + 0.01
                 * float(
@@ -121,86 +121,60 @@ def _get_lexico_bound(self, metric, mode):
 
     def update_priority(self, eci: Optional[float] = 0):
         if self.lexico_objectives and self._is_ls:
-            feasible_flag = True
             for k_metric, k_mode in zip(
                 self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
             ):
-                k_bound = _get_lexico_bound(k_metric, k_mode)
-                if self.obj_best1[k_metric] < k_bound:
-                    self.priority[k_metric] = self.obj_best1[k_metric]
-                    continue
-                elif self.obj_best1[k_metric] < self.f_best[k_metric]:
-                    raise ValueError(
-                        "Bug exists: Performance of a specific search thread is not possible to be better than global best.")
-                else:
-                    start_point = self.obj_best1[k_metric] if feasible_flag else self.f_worst[k_metric]
-                    if start_point <= k_bound:
-                        continue
-                    elif start_point > k_bound and start_point - eci * self.speed[k_metric] < k_bound:
-                        eci -= (start_point - k_bound) / self.speed[k_metric]
-                        self.priority[k_metric] = k_bound
-                    elif eci == 0:
-                        self.priority[k_metric] = self.f_worst[k_metric]
-                    else:
-                        self.priority[k_metric] = start_point - eci * self.speed[k_metric] 
-                        eci = 0
-                    feasible_flag = False
+                self.priority[k_metric] = eci * self.speed[k_metric] - self.obj_best1[k_metric] 
         else:
             self.priority = eci * self.speed - self.obj_best1
 
-    def update_eci(self, metric_target: Union[float, dict], max_speed: Optional[Union[float, dict]], f_best: Optional[dict], f_worst: Optional[dict]):
+    def update_eci(self, metric_target: float, max_speed: Optional[float] = np.inf, min_speed: Optional[float] = 1e-9):
         # calculate eci: estimated cost for improvement over metric_target
-        if self.lexico_objectives is None or not self._is_ls:  
-            best_obj = metric_target * self._metric_op
+        # if lexico, metric_target = _f_best[_metric_1st], else global best 
+        _metric_1st = self.lexico_objectives["metrics"][0]
+        if self.lexico_objectives is None:
+            _metric_op = self._metric_op
             if not self.speed:
+               self.speed = max_speed 
+        else:
+            _metric_op = 1 if self.lexico_objectives["modes"][_metric_1st] == "min" else -1
+            if self.speed[_metric_1st] == 0:
                 self.speed = max_speed
-            self.eci = max(
-                self.cost_total - self.cost_best1, self.cost_best1 - self.cost_best2
-            )
-            if self.obj_best1 > best_obj and self.speed > 0:
-                self.eci = max(self.eci, 2 * (self.obj_best1 - best_obj) / self.speed)
-        else:  # Optimization lexicographically
-            self.eci = max(
-                self.cost_total - self.cost_best1, self.cost_best1 - self.cost_best2
-            )
-            eci_last = 0
-            feasible_flag = True
-            # TODO: finish 3/31
-            for k_metric, k_mode in zip(
-                self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
-            ):
-                if not self.speed[k_metric]:
-                    self.speed[k_metric] = max_speed
-                k_bound = _get_lexico_bound(k_metric, k_mode)
-                if self.obj_best1[k_metric] < k_bound:
-                    continue
-                elif self.obj_best1[k_metric] < self.f_best[k_metric]:
-                    raise ValueError(
-                        "Bug exists: Performance of a specific search thread is not possible to be better than global best.")
-                else:               
-                    if self.speed[k_metric] > 0: # check again in which cases speed[k_metric] <= 0
-                        start_point = self.obj_best1[k_metric] if feasible_flag else self.f_worst[k_metric]
-                        eci_last + = (start_point - k_bound) / self.speed[k_metric]
-                    feasible_flag = False
-            self.eci = max(self.eci, 2 * eci_last)
-
+            elif self.speed[_metric_1st] == -1:
+                self.speed = min_speed
+        best_obj = metric_target * _metric_op
+        self.eci = max(self.cost_total - self.cost_best1, self.cost_best1 - self.cost_best2)
+        # get "obj_best1" and "speed"
+        obj_best1 = self.obj_best1 if not self.lexico_objectives else self.obj_best1[_metric_1st]
+        speed = self.speed if not self.lexico_objectives else self.speed[_metric_1st]
+        if obj_best1 > best_obj and speed > 0:
+            self.eci = max(self.eci, 2 * (self.obj_best1 - best_obj) / self.speed)
+        
     def _update_speed(self):
         # calculate speed; use 0 for invalid speed temporarily
         if self.lexico_objectives is None and self.obj_best2 > self.obj_best1:
-            # discount the speed if there are unfinished trials
             self.speed = (
-                (self.obj_best2 - self.obj_best1)
+                (self.obj_best2 - self.obj_best1) / self.running / (max(self.cost_total - self.cost_best2, self._eps))
+            )
+        elif self.lexico_objectives != None and self.obj_best2 != self.obj_best1:
+            op_dimension = self._search_alg.op_dimension
+            op_index = self.lexico_objectives["metrics"].index(op_dimension)
+            metrics_length = len(self.lexico_objectives["metrics"])
+            self.speed[op_dimension] = (
+                (self.obj_best2[op_dimension] - self.obj_best1[op_dimension])
                 / self.running
                 / (max(self.cost_total - self.cost_best2, self._eps))
             )
-            for i in range(0, len(metrics)):
-                if i != op_index:
+            for i in range(0, metrics_length):
+                if i < op_index:
+                    self.speed[self.lexico_objectives["metrics"][i]] = -1
+                elif i > op_index:
                     self.speed[self.lexico_objectives["metrics"][i]] = 0
         else:
             if self.lexico_objectives is None:
                 self.speed = 0
             else:
-                for i in range(0, len(metrics)):
+                for i in range(0, metrics_length):
                     self.speed[self.lexico_objectives["metrics"][i]] = 0
 
     def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
@@ -220,7 +194,7 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
                 # init config is not proposed by self._search_alg
                 # under this thread
                 self._init_config = False
-        if result:
+        if result: # check
             self.cost_last = result.get(self.cost_attr, 1)
             self.cost_total += self.cost_last
             if self._search_alg.metric in result:
@@ -241,10 +215,7 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
                     self.obj_best1 = obj
                     self.cost_best = self.cost_last
                     self.best_result = result
-            if self.lexico_objectives is None:
-                # TODO: Improve this behavior. When lexico_objectives is provided to CFO,
-                # related variables are not callable.
-                self._update_speed()
+            self._update_speed()
         self.running -= 1
         assert self.running >= 0
 
@@ -262,7 +233,6 @@ def on_trial_result(self, trial_id: str, result: Dict):
         new_cost = result.get(self.cost_attr, 1)
         if self.cost_last < new_cost:
             self.cost_last = new_cost
-            # self._update_speed()
 
     @property
     def converged(self) -> bool:
diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py
index df1aee19c2..475186a063 100644
--- a/flaml/tune/tune.py
+++ b/flaml/tune/tune.py
@@ -83,7 +83,7 @@ def lexico_best(self, trials):
             k_values = np.array(histories[k_metric])
             k_target = (
                 -self.lexico_objectives["targets"][k_metric]
-                if k_mode == "max"
+                if  == "max"
                 else self.lexico_objectives["targets"][k_metric]
             )
             feasible_value = k_values.take(feasible_index)

From f1de480f83290831170a579159bb3d2bea68ac85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Wed, 14 Jun 2023 14:40:47 +0800
Subject: [PATCH 03/15] update

---
 flaml/tune/searcher/blendsearch.py   | 75 +++++++++++++++++++---------
 flaml/tune/searcher/flow2.py         |  2 +-
 flaml/tune/searcher/search_thread.py |  2 +-
 3 files changed, 53 insertions(+), 26 deletions(-)

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index b44bedf264..051203da8f 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -480,7 +480,7 @@ def metric_target(self):
     def is_ls_ever_converged(self):
         return self._is_ls_ever_converged
 
-    def on_trial_complete(  # check
+    def on_trial_complete(  
         self, trial_id: str, result: Optional[Dict] = None, error: bool = False
     ):
         """search thread updater and cleaner."""
@@ -637,14 +637,21 @@ def _update_admissible_region(
                 elif value < admissible_min[key]:
                     admissible_min[key] = value
 
-    def _create_condition(self, result: Dict) -> bool:  # check
+    def _create_condition(self, result: Dict) -> bool:  
         """create thread condition"""
         if len(self._search_thread_pool) < 2:
             return True
-        obj_median = np.median([thread.obj_best1 for id, thread in self._search_thread_pool.items() if id])
-        return result[self._ls.metric] * self._ls.metric_op < obj_median
+        if not self.lexico_objectives:
+            obj_median = np.median([thread.obj_best1 for id, thread in self._search_thread_pool.items() if id])
+            return result[self._ls.metric] * self._ls.metric_op < obj_median
+        else:
+            thread_pools = [thread.obj_best1 for id, thread in self._search_thread_pool.items() if id]
+            self._lexico_sort(thread_pools)
+            obj_median = thread_pools[round(len(thread_pools)/2)]
+            result = self._unify_op(result)
+            return self._lexico_inferior(obj_median, result)
 
-    def _clean(self, thread_id: int):  # check
+    def _clean(self, thread_id: int):  
         """delete thread and increase admissible region if converged,
         merge local threads if they are close
         """
@@ -668,7 +675,7 @@ def _clean(self, thread_id: int):  # check
                 self._ls_bound_max,
                 self._search_thread_pool[thread_id].space,
             )
-            if self._candidate_start_points:
+            if self._candidate_start_points and not self.lexico_objectives:
                 if not self._started_from_given:
                     # remove start points whose perf is worse than the converged
                     obj = self._search_thread_pool[thread_id].obj_best1
@@ -687,14 +694,16 @@ def _clean(self, thread_id: int):  # check
         if create_new:
             self._create_thread_from_best_candidate()
 
-    def _create_thread_from_best_candidate(self):  # check
+    def _create_thread_from_best_candidate(self):  
         # find the best start point
         best_trial_id = None
         obj_best = None
         for trial_id, r in self._candidate_start_points.items():
-            if r and (best_trial_id is None or r[self._ls.metric] * self._ls.metric_op < obj_best):
-                best_trial_id = trial_id
-                obj_best = r[self._ls.metric] * self._ls.metric_op
+            if r: 
+                r = self._unify_op(r)
+                if best_trial_id is None or self._lexico_inferior(obj_best, r):
+                    best_trial_id = trial_id
+                    obj_best = r
         if best_trial_id:
             # create a new thread
             config = {}
@@ -720,36 +729,54 @@ def _expand_admissible_region(self, lower, upper, space):
                 upper[key] += self._ls.STEPSIZE
                 lower[key] -= self._ls.STEPSIZE
 
-    def _priority_inferior(self, priority_1: Union[dict, float], priority_2: Union[dict, float]) -> bool:
+    def _lexico_inferior(self, obj_1: Union[dict, float], obj_2: Union[dict, float]) -> bool:
         if self.lexico_objectives:
             for k_metric, k_mode in zip(
                 self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
             ):
                 bound = self._get_lexico_bound(k_metric, k_mode)
-                if (priority_1[k_metric] < bound) and (priority_2[k_metric] < bound):
+                if (obj_1[k_metric] < bound) and (obj_2[k_metric] < bound):
                     continue
-                elif priority_1[k_metric] < priority_2[k_metric]:
-                    return True
-                else:
+                elif obj_1[k_metric] < obj_2[k_metric]:
                     return False
+                else:
+                    return True
             for k_metr in self.lexico_objectives["metrics"]:
-                if priority_1[k_metr] == priority_2[k_metr]:
+                if obj_1[k_metr] == obj_2[k_metr]:
                     continue
-                elif priority_1[k_metr] < priority_2[k_metr]:
-                    return True
-                else:
+                elif obj_1[k_metr] < obj_2[k_metr]:
                     return False
+                else:
+                    return True
         else:
-            if priority_1 > priority_2: 
+            if obj_1 > obj_2: 
                 return True
             else:
                 return False
+    
+    def _unify_op(self, result: Union[dict, float]):
+        if isinstance(result, dict):
+            for k_metric, k_mode in zip(
+                self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
+            ):
+                result[k_metric] = -1 * result[k_metric] if k_mode == "max" else result[k_metric]
+        else:
+            result[self._ls.metric] = result[self._ls.metric] * self._ls.metric_op
+        return result
+
+    
+    def _lexico_sort(self, arr: list):
+        n = len(arr)
+        for i in range(n):
+            for j in range(0, n - i - 1):
+                if self._lexico_(arr[j], arr[j + 1]):
+                    arr[j], arr[j + 1] = arr[j + 1], arr[j]
             
-    def _inferior(self, id1: int, id2: int) -> bool:  # check
+    def _inferior(self, id1: int, id2: int) -> bool:  
         """whether thread id1 is inferior to id2"""
         t1 = self._search_thread_pool[id1]
         t2 = self._search_thread_pool[id2]
-        if t1.obj_best1 < t2.obj_best2:
+        if self._lexico_inferior(t2.obj_best2, t1.obj_best1):
             return False
         elif t1.resource and t1.resource < t2.resource:
             return False
@@ -1009,10 +1036,10 @@ def _select_thread(self) -> Tuple:
         for thread_id, thread in self._search_thread_pool.items():
             if thread_id and thread.can_suggest:
                 priority = thread.priority
-                if self._priority_inferior(priority, priority1):
+                if not self._lexico_inferior(priority, priority1):
                     priority1 = priority
                     top_thread_id = thread_id
-                if self._priority_inferior(priority, priority2) or backup_thread_id == 0:
+                if not self._lexico_inferior(priority, priority2) or backup_thread_id == 0:
                     priority2 = priority
                     backup_thread_id = thread_id
         return top_thread_id, backup_thread_id
diff --git a/flaml/tune/searcher/flow2.py b/flaml/tune/searcher/flow2.py
index 4eeea4d8d9..6b75a3edaf 100644
--- a/flaml/tune/searcher/flow2.py
+++ b/flaml/tune/searcher/flow2.py
@@ -296,7 +296,7 @@ def create(self, init_config: Dict, obj: float, cost: float, space: Dict) -> Sea
             flow2.best_obj = {}
             for k, v in obj.items():
                 flow2.best_obj[k] = (
-                    -v if self.lexico_objectives["modes"][self.lexico_objectives["metrics"].index(k)] == "max" else v
+                    -1 * v if self.lexico_objectives["modes"][self.lexico_objectives["metrics"].index(k)] == "max" else v
                 )
         else:
             flow2.best_obj = obj * self.metric_op  # minimize internally
diff --git a/flaml/tune/searcher/search_thread.py b/flaml/tune/searcher/search_thread.py
index 3cadac6a89..114a170a58 100644
--- a/flaml/tune/searcher/search_thread.py
+++ b/flaml/tune/searcher/search_thread.py
@@ -194,7 +194,7 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
                 # init config is not proposed by self._search_alg
                 # under this thread
                 self._init_config = False
-        if result: # check
+        if result: 
             self.cost_last = result.get(self.cost_attr, 1)
             self.cost_total += self.cost_last
             if self._search_alg.metric in result:

From 321ef3e2a1461d52d100b2e02aff7f4505ce8c9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Thu, 15 Jun 2023 15:24:39 +0800
Subject: [PATCH 04/15] update

---
 flaml/tune/searcher/blendsearch.py   | 241 +++++++++++++++++----------
 flaml/tune/searcher/search_thread.py |  88 +++++-----
 flaml/tune/tune.py                   |   6 +-
 test.py                              | 206 +++++++++++++++++++++++
 4 files changed, 401 insertions(+), 140 deletions(-)
 create mode 100644 test.py

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index 051203da8f..b90421c0fa 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -35,6 +35,22 @@
 logger = logging.getLogger(__name__)
 
 
+class Lexico_GlobalSearch(GlobalSearch):
+    def set_search_properties(
+        self, metric: Optional[Union[str, List[str]]], mode: Optional[Union[str, List[str]]], config: Dict
+    ) -> bool:
+        if self._space:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+        self._setup_study(mode)
+        return True
+
+
 class BlendSearch(Searcher):
     """class for BlendSearch algorithm."""
 
@@ -151,6 +167,7 @@ def __init__(
         """
         self._eps = SEARCH_THREAD_EPS
         self._input_cost_attr = cost_attr
+        self.lexico_objectives = lexico_objectives
         if cost_attr == "auto":
             if time_budget_s is not None:
                 self.cost_attr = TIME_TOTAL_S
@@ -161,7 +178,11 @@ def __init__(
             self.cost_attr = cost_attr
             self._cost_budget = cost_budget
         self.penalty = PENALTY  # penalty term for constraints
-        self._metric, self._mode = metric, mode
+        if not self.lexico_objectives:
+            self._metric, self._mode = metric, mode
+        else:
+            self._metric = self.lexico_objectives["metrics"][0]
+            self._mode = self.lexico_objectives["modes"][0]
         self._use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
         self.lexico_objectives = lexico_objectives
         self._histories = None
@@ -197,9 +218,15 @@ def __init__(
         self._config_constraints = config_constraints
         self._metric_constraints = metric_constraints
         if metric_constraints:
-            assert all(x[1] in ["<=", ">="] for x in metric_constraints), "sign of metric constraints must be <= or >=."
-            # metric modified by lagrange
-            metric += self.lagrange
+            if self.lexico_objectives:
+                self._metric_constraints = None
+                logger.info("Do not support providing metric_constraints in lexicographic optimization for now.")
+            else:
+                assert all(
+                    x[1] in ["<=", ">="] for x in metric_constraints
+                ), "sign of metric constraints must be <= or >=."
+                # metric modified by lagrange
+                metric += self.lagrange
         self._cat_hp_cost = cat_hp_cost or {}
         if space:
             add_cost_to_space(space, init_config, self._cat_hp_cost)
@@ -231,28 +258,62 @@ def __init__(
             if experimental:
                 import optuna as ot
 
-                sampler = ot.samplers.TPESampler(seed=gs_seed, multivariate=True, group=True)
+                if not self.lexico_objectives:
+                    sampler = ot.samplers.TPESampler(seed=gs_seed, multivariate=True, group=True)
+                else:
+                    n_startup_trials = 11 * len(self.lexico_objectives["metrics"]) - 1
+                    sampler = ot.samplers.MOTPESampler(
+                        seed=gs_seed, n_startup_trials=n_startup_trials, n_ehvi_candidates=24
+                    )
+
             else:
                 sampler = None
+            if self.lexico_objectives:
+                metric, mode = [], []
+                for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
+                    metric.append(k_metric)
+                    mode.append(k_mode)
             try:
                 assert evaluated_rewards
-                self._gs = GlobalSearch(
-                    space=gs_space,
-                    metric=metric,
-                    mode=mode,
-                    seed=gs_seed,
-                    sampler=sampler,
-                    points_to_evaluate=self._evaluated_points,
-                    evaluated_rewards=evaluated_rewards,
-                )
+                if self.lexico_objectives:
+                    self._gs = Lexico_GlobalSearch(
+                        space=gs_space,
+                        metric=metric,
+                        mode=mode,
+                        seed=gs_seed,
+                        sampler=sampler,
+                        points_to_evaluate=self._evaluated_points,
+                        evaluated_rewards=evaluated_rewards,
+                    )
+                    Lexico_GlobalSearch.lexico_objectives = self.lexico_objective
+                else:
+                    self._gs = GlobalSearch(
+                        space=gs_space,
+                        metric=metric,
+                        mode=mode,
+                        seed=gs_seed,
+                        sampler=sampler,
+                        points_to_evaluate=self._evaluated_points,
+                        evaluated_rewards=evaluated_rewards,
+                    )
             except (AssertionError, ValueError):
-                self._gs = GlobalSearch(
-                    space=gs_space,
-                    metric=metric,
-                    mode=mode,
-                    seed=gs_seed,
-                    sampler=sampler,
-                )
+                if self.lexico_objectives:
+                    self._gs = Lexico_GlobalSearch(
+                        space=gs_space,
+                        metric=metric,
+                        mode=mode,
+                        seed=gs_seed,
+                        sampler=sampler,
+                    )
+                    Lexico_GlobalSearch.lexico_objectives = self.lexico_objectives
+                else:
+                    self._gs = GlobalSearch(
+                        space=gs_space,
+                        metric=metric,
+                        mode=mode,
+                        seed=gs_seed,
+                        sampler=sampler,
+                    )
             self._gs.space = space
         else:
             self._gs = None
@@ -278,20 +339,13 @@ def update_fbest(
             feasible_value = k_values.take(feasible_index)
             self._f_best[k_metric] = np.min(feasible_value)
             if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
-                tolerance_bound = (
-                    self._f_best[k_metric]
-                    + self.lexico_objectives["tolerances"][k_metric]
-                )
+                tolerance_bound = self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
             else:
                 assert (
                     self.lexico_objectives["tolerances"][k_metric][-1] == "%"
                 ), "String tolerance of {} should use %% as the suffix".format(k_metric)
                 tolerance_bound = self._f_best[k_metric] * (
-                    1
-                    + 0.01
-                    * float(
-                        self.lexico_objectives["tolerances"][k_metric].replace("%", "")
-                    )
+                    1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))
                 )
             feasible_index_filter = np.where(
                 feasible_value
@@ -336,12 +390,20 @@ def set_search_properties(
                 # reset search when metric or mode changed
                 self._ls.set_search_properties(metric, mode)
                 if self._gs is not None:
-                    self._gs = GlobalSearch(
-                        space=self._gs._space,
-                        metric=metric,
-                        mode=mode,
-                        seed=self._gs_seed,
-                    )
+                    if self.lexico_objectives:
+                        self._gs = Lexico_GlobalSearch(
+                            space=self._gs._space,
+                            metric=metric,
+                            mode=mode,
+                            seed=self._gs_seed,
+                        )
+                    else:
+                        self._gs = GlobalSearch(
+                            space=self._gs._space,
+                            metric=metric,
+                            mode=mode,
+                            seed=self._gs_seed,
+                        )
                     self._gs.space = self._ls.space
                 self._init_search()
         if spec:
@@ -409,7 +471,7 @@ def _init_search(self):
         self._gs_admissible_min = self._ls_bound_min.copy()
         self._gs_admissible_max = self._ls_bound_max.copy()
 
-        if self._metric_constraints:  
+        if self._metric_constraints:
             self._metric_constraint_satisfied = False
             self._metric_constraint_penalty = [self.penalty for _ in self._metric_constraints]
         else:
@@ -443,31 +505,17 @@ def restore(self, checkpoint_path: str):
         self._set_deadline()
 
     def _get_lexico_bound(self, metric, mode):
-
         k_target = (
-            self.lexico_objectives["targets"][metric]
-            if mode == "min"
-            else -self.lexico_objectives["targets"][metric]
+            self.lexico_objectives["targets"][metric] if mode == "min" else -self.lexico_objectives["targets"][metric]
         )
         if not isinstance(self.lexico_objectives["tolerances"][metric], str):
-            tolerance_bound = (
-                self._f_best[metric]
-                + self.lexico_objectives["tolerances"][metric]
-            )
+            tolerance_bound = self._f_best[metric] + self.lexico_objectives["tolerances"][metric]
         else:
             assert (
                 self.lexico_objectives["tolerances"][metric][-1] == "%"
-            ), "String tolerance of {} should use %% as the suffix".format(
-                metric
-            )
-            tolerance_bound = self.f_best[metric] * (
-                1
-                + 0.01
-                * float(
-                    self.lexico_objectives["tolerances"][metric].replace(
-                        "%", ""
-                    )
-                )
+            ), "String tolerance of {} should use %% as the suffix".format(metric)
+            tolerance_bound = self._f_best[metric] * (
+                1 + 0.01 * float(self.lexico_objectives["tolerances"][metric].replace("%", ""))
             )
         bound = max(tolerance_bound, k_target)
         return bound
@@ -480,12 +528,10 @@ def metric_target(self):
     def is_ls_ever_converged(self):
         return self._is_ls_ever_converged
 
-    def on_trial_complete(  
-        self, trial_id: str, result: Optional[Dict] = None, error: bool = False
-    ):
+    def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
         """search thread updater and cleaner."""
         metric_constraint_satisfied = True
-        if result and not error and self._metric_constraints:  # remove check
+        if result and not error and self._metric_constraints:
             # account for metric constraints if any
             objective = result[self._metric]
             for i, constraint in enumerate(self._metric_constraints):
@@ -554,7 +600,7 @@ def on_trial_complete(
                     # thread creator
                     thread_id = self._thread_count
                     self._started_from_given = self._candidate_start_points and trial_id in self._candidate_start_points
-                    if self._started_from_given:
+                    if self._started_from_given:  # check
                         del self._candidate_start_points[trial_id]
                     else:
                         self._started_from_low_cost = True
@@ -637,7 +683,7 @@ def _update_admissible_region(
                 elif value < admissible_min[key]:
                     admissible_min[key] = value
 
-    def _create_condition(self, result: Dict) -> bool:  
+    def _create_condition(self, result: Dict) -> bool:
         """create thread condition"""
         if len(self._search_thread_pool) < 2:
             return True
@@ -647,11 +693,11 @@ def _create_condition(self, result: Dict) -> bool:
         else:
             thread_pools = [thread.obj_best1 for id, thread in self._search_thread_pool.items() if id]
             self._lexico_sort(thread_pools)
-            obj_median = thread_pools[round(len(thread_pools)/2)]
+            obj_median = thread_pools[round(len(thread_pools) / 2)]
             result = self._unify_op(result)
             return self._lexico_inferior(obj_median, result)
 
-    def _clean(self, thread_id: int):  
+    def _clean(self, thread_id: int):
         """delete thread and increase admissible region if converged,
         merge local threads if they are close
         """
@@ -694,12 +740,12 @@ def _clean(self, thread_id: int):
         if create_new:
             self._create_thread_from_best_candidate()
 
-    def _create_thread_from_best_candidate(self):  
+    def _create_thread_from_best_candidate(self):
         # find the best start point
         best_trial_id = None
         obj_best = None
         for trial_id, r in self._candidate_start_points.items():
-            if r: 
+            if r:
                 r = self._unify_op(r)
                 if best_trial_id is None or self._lexico_inferior(obj_best, r):
                     best_trial_id = trial_id
@@ -731,9 +777,7 @@ def _expand_admissible_region(self, lower, upper, space):
 
     def _lexico_inferior(self, obj_1: Union[dict, float], obj_2: Union[dict, float]) -> bool:
         if self.lexico_objectives:
-            for k_metric, k_mode in zip(
-                self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
-            ):
+            for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
                 bound = self._get_lexico_bound(k_metric, k_mode)
                 if (obj_1[k_metric] < bound) and (obj_2[k_metric] < bound):
                     continue
@@ -749,30 +793,27 @@ def _lexico_inferior(self, obj_1: Union[dict, float], obj_2: Union[dict, float])
                 else:
                     return True
         else:
-            if obj_1 > obj_2: 
+            if obj_1 > obj_2:
                 return True
             else:
                 return False
-    
+
     def _unify_op(self, result: Union[dict, float]):
         if isinstance(result, dict):
-            for k_metric, k_mode in zip(
-                self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
-            ):
+            for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
                 result[k_metric] = -1 * result[k_metric] if k_mode == "max" else result[k_metric]
         else:
             result[self._ls.metric] = result[self._ls.metric] * self._ls.metric_op
         return result
 
-    
     def _lexico_sort(self, arr: list):
         n = len(arr)
         for i in range(n):
             for j in range(0, n - i - 1):
                 if self._lexico_(arr[j], arr[j + 1]):
                     arr[j], arr[j + 1] = arr[j + 1], arr[j]
-            
-    def _inferior(self, id1: int, id2: int) -> bool:  
+
+    def _inferior(self, id1: int, id2: int) -> bool:
         """whether thread id1 is inferior to id2"""
         t1 = self._search_thread_pool[id1]
         t2 = self._search_thread_pool[id2]
@@ -883,6 +924,10 @@ def suggest(self, trial_id: str) -> Optional[Dict]:
             self._subspace[trial_id] = space
         else:  # use init config
             if self._candidate_start_points is not None and self._points_to_evaluate:
+                if self.lexico_objectives:
+                    raise NotImplementedError(
+                        "It doesn't support providing points_to_evaluate in lexicographic optimization for now."
+                    )
                 self._candidate_start_points[trial_id] = None
             reward = None
             if self._points_to_evaluate:
@@ -941,10 +986,18 @@ def _violate_config_constriants(self, config, config_signature):
                 or sign == "<"
                 and value > threshold
             ):
-                self._result[config_signature] = {
-                    self._metric: np.inf * self._ls.metric_op,
-                    "time_total_s": 1,
-                }
+                if not self.lexico_objectives:
+                    self._result[config_signature] = {
+                        self._metric: np.inf * self._ls.metric_op,
+                        "time_total_s": 1,
+                    }
+                else:
+                    for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
+                        self._result[config_signature] = {}
+                        self._result[config_signature][k_metric] = {
+                            self._metric: np.inf * -1 if k_mode == "max" else np.inf
+                        }
+                        self._result[config_signature]["time_total_s"] = 1
                 return True
         return False
 
@@ -1021,9 +1074,9 @@ def _select_thread(self) -> Tuple:
         for thread in self._search_thread_pool.values():
             if self.lexico_objectives is None:
                 thread.update_eci(self._metric_target, max_speed, min_speed)
-            else: 
+            else:
                 _metric_1st = self.lexico_objectives["metrics"][0]
-                _op_1st = self.lexico_objectives["modes"][_metric_1st]
+                _op_1st = self.lexico_objectives["modes"][0]
                 _lexico_target = self._f_best[_metric_1st] if _op_1st == "min" else -1 * self._f_best[_metric_1st]
                 thread.update_eci(_lexico_target, max_speed, min_speed)
             if thread.eci < min_eci:
@@ -1201,12 +1254,20 @@ def update_search_space(self, search_space):
             lexico_objectives=self.lexico_objectives,
         )
         if self._gs is not None:
-            self._gs = GlobalSearch(
-                space=config,
-                metric=self._metric,
-                mode=self._mode,
-                sampler=self._gs._sampler,
-            )
+            if self.lexico_objectives:
+                self._gs = Lexico_GlobalSearch(
+                    space=config,
+                    metric=self._metric,
+                    mode=self._mode,
+                    sampler=self._gs._sampler,
+                )
+            else:
+                self._gs = GlobalSearch(
+                    space=config,
+                    metric=self._metric,
+                    mode=self._mode,
+                    sampler=self._gs._sampler,
+                )
             self._gs.space = config
         self._init_search()
 
@@ -1272,4 +1333,4 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
         return
 
     def on_trial_result(self, trial_id: str, result: Dict):
-        return
\ No newline at end of file
+        return
diff --git a/flaml/tune/searcher/search_thread.py b/flaml/tune/searcher/search_thread.py
index 114a170a58..ea8b79e789 100644
--- a/flaml/tune/searcher/search_thread.py
+++ b/flaml/tune/searcher/search_thread.py
@@ -43,22 +43,23 @@ def __init__(
         self.cost_best2 = 0
         self.lexico_objectives = getattr(self._search_alg, "lexico_objectives", None)
 
-        # get obj 1 and obj 2
-        if self._is_ls and self.lexico_objectives:
+        if self.lexico_objectives:
+            # set 1st to 0 others to -1?
             self.obj_best1 = self.obj_best2 = {}
             for k in self.lexico_objectives["metrics"]:
-                self.obj_best1[k] = self.obj_best2[k] = np.inf if getattr(
-                    search_alg, "best_obj", None) is None else search_alg.best_obj[k]
+                self.obj_best1[k] = self.obj_best2[k] = (
+                    np.inf if getattr(search_alg, "best_obj", None) is None else search_alg.best_obj[k]
+                )
         else:
-            self.obj_best1 = self.obj_best2 = getattr(
-                search_alg, "best_obj", np.inf
-            )  # inherently minimize
+            self.obj_best1 = self.obj_best2 = getattr(search_alg, "best_obj", np.inf)  # inherently minimize
 
         self.best_result = None
         # eci: estimated cost for improvement
         self.eci = self.cost_best
-        if self._is_ls and self.lexico_objectives:
-            self.priority = self.speed = {}
+        if self.lexico_objectives:
+            self.priority, self.speed = {}, {}
+            for k_metric in self.lexico_objectives["metrics"]:
+                self.priority[k_metric] = self.speed[k_metric] = 0
         else:
             self.priority = self.speed = 0
         self._init_config = True
@@ -90,54 +91,38 @@ def suggest(self, trial_id: str) -> Optional[Dict]:
         return config
 
     def _get_lexico_bound(self, metric, mode):
-
         k_target = (
-            self.lexico_objectives["targets"][metric]
-            if mode == "min"
-            else -self.lexico_objectives["targets"][metric]
+            self.lexico_objectives["targets"][metric] if mode == "min" else -self.lexico_objectives["targets"][metric]
         )
         if not isinstance(self.lexico_objectives["tolerances"][metric], str):
-            tolerance_bound = (
-                self._f_best[metric]
-                + self.lexico_objectives["tolerances"][metric]
-            )
+            tolerance_bound = self._f_best[metric] + self.lexico_objectives["tolerances"][metric]
         else:
             assert (
                 self.lexico_objectives["tolerances"][metric][-1] == "%"
-            ), "String tolerance of {} should use %% as the suffix".format(
-                metric
-            )
+            ), "String tolerance of {} should use %% as the suffix".format(metric)
             tolerance_bound = self._f_best[metric] * (
-                1
-                + 0.01
-                * float(
-                    self.lexico_objectives["tolerances"][metric].replace(
-                        "%", ""
-                    )
-                )
+                1 + 0.01 * float(self.lexico_objectives["tolerances"][metric].replace("%", ""))
             )
         bound = max(tolerance_bound, k_target)
         return bound
 
     def update_priority(self, eci: Optional[float] = 0):
-        if self.lexico_objectives and self._is_ls:
-            for k_metric, k_mode in zip(
-                self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
-            ):
-                self.priority[k_metric] = eci * self.speed[k_metric] - self.obj_best1[k_metric] 
+        if self.lexico_objectives:
+            for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
+                self.priority[k_metric] = eci * self.speed[k_metric] - self.obj_best1[k_metric]
         else:
             self.priority = eci * self.speed - self.obj_best1
 
     def update_eci(self, metric_target: float, max_speed: Optional[float] = np.inf, min_speed: Optional[float] = 1e-9):
         # calculate eci: estimated cost for improvement over metric_target
-        # if lexico, metric_target = _f_best[_metric_1st], else global best 
-        _metric_1st = self.lexico_objectives["metrics"][0]
+        # if lexico, metric_target = _f_best[_metric_1st], else global best
         if self.lexico_objectives is None:
             _metric_op = self._metric_op
             if not self.speed:
-               self.speed = max_speed 
+                self.speed = max_speed
         else:
-            _metric_op = 1 if self.lexico_objectives["modes"][_metric_1st] == "min" else -1
+            _metric_1st = self.lexico_objectives["metrics"][0]
+            _metric_op = 1 if self.lexico_objectives["modes"][0] == "min" else -1
             if self.speed[_metric_1st] == 0:
                 self.speed = max_speed
             elif self.speed[_metric_1st] == -1:
@@ -149,14 +134,14 @@ def update_eci(self, metric_target: float, max_speed: Optional[float] = np.inf,
         speed = self.speed if not self.lexico_objectives else self.speed[_metric_1st]
         if obj_best1 > best_obj and speed > 0:
             self.eci = max(self.eci, 2 * (self.obj_best1 - best_obj) / self.speed)
-        
+
     def _update_speed(self):
         # calculate speed; use 0 for invalid speed temporarily
         if self.lexico_objectives is None and self.obj_best2 > self.obj_best1:
             self.speed = (
                 (self.obj_best2 - self.obj_best1) / self.running / (max(self.cost_total - self.cost_best2, self._eps))
             )
-        elif self.lexico_objectives != None and self.obj_best2 != self.obj_best1:
+        elif self.lexico_objectives is not None and self.obj_best2 != self.obj_best1:
             op_dimension = self._search_alg.op_dimension
             op_index = self.lexico_objectives["metrics"].index(op_dimension)
             metrics_length = len(self.lexico_objectives["metrics"])
@@ -174,7 +159,7 @@ def _update_speed(self):
             if self.lexico_objectives is None:
                 self.speed = 0
             else:
-                for i in range(0, metrics_length):
+                for i in range(0, len(self.lexico_objectives["metrics"])):
                     self.speed[self.lexico_objectives["metrics"][i]] = 0
 
     def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
@@ -194,24 +179,35 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
                 # init config is not proposed by self._search_alg
                 # under this thread
                 self._init_config = False
-        if result: 
+        if result:
             self.cost_last = result.get(self.cost_attr, 1)
             self.cost_total += self.cost_last
-            if self._search_alg.metric in result:
+            if self.lexico_objectives is None:
+                feasible_condition = self._search_alg.metric in result
+            else:
+                feasible_condition = all(x in result for x in self._search_alg.metric)
+            if feasible_condition:
                 if self.lexico_objectives is None:
                     obj = result[self._search_alg.metric] * self._metric_op
                 else:
                     obj = {}
-                    for k, m in zip(self._search_alg.lexico_objectives["metrics"], self._search_alg.lexico_objectives["modes"]):
+                    for k, m in zip(
+                        self._search_alg.lexico_objectives["metrics"], self._search_alg.lexico_objectives["modes"]
+                    ):
                         obj[k] = -result[k] if m == "max" else result[k]
-                if self.best_result is None or (self.lexico_objectives is None and obj < self.obj_best1) or (self.lexico_objectives is not None and obj == self._search_alg.best_obj):
+                if (
+                    self.best_result is None
+                    or (self.lexico_objectives is None and obj < self.obj_best1)
+                    or (self.lexico_objectives is not None and obj == self._search_alg.best_obj)
+                ):
                     self.cost_best2 = self.cost_best1
                     self.cost_best1 = self.cost_total
                     if self.lexico_objectives is None:
                         self.obj_best2 = obj if np.isinf(self.obj_best1) else self.obj_best1
                     else:
-                        self.obj_best2 = obj if np.isinf(
-                            self.obj_best1[self.lexico_objectives["metrics"][0]]) else self.obj_best1
+                        self.obj_best2 = (
+                            obj if np.isinf(self.obj_best1[self.lexico_objectives["metrics"][0]]) else self.obj_best1
+                        )
                     self.obj_best1 = obj
                     self.cost_best = self.cost_last
                     self.best_result = result
@@ -249,4 +245,4 @@ def reach(self, thread) -> bool:
     @property
     def can_suggest(self) -> bool:
         """Whether the thread can suggest new configs."""
-        return self._search_alg.can_suggest
\ No newline at end of file
+        return self._search_alg.can_suggest
diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py
index 475186a063..031d828624 100644
--- a/flaml/tune/tune.py
+++ b/flaml/tune/tune.py
@@ -83,7 +83,7 @@ def lexico_best(self, trials):
             k_values = np.array(histories[k_metric])
             k_target = (
                 -self.lexico_objectives["targets"][k_metric]
-                if  == "max"
+                if k_mode == "max"
                 else self.lexico_objectives["targets"][k_metric]
             )
             feasible_value = k_values.take(feasible_index)
@@ -465,9 +465,7 @@ def easy_objective(config):
     from .searcher.blendsearch import BlendSearch, CFO
 
     if lexico_objectives is not None:
-        logger.warning(
-            "If lexico_objectives is not None, search_alg is forced to be CFO or Blendsearch" 
-        )
+        logger.warning("If lexico_objectives is not None, search_alg is forced to be CFO or Blendsearch")
         search_alg = None
     if search_alg is None:
         flaml_scheduler_resource_attr = (
diff --git a/test.py b/test.py
new file mode 100644
index 0000000000..04ae488ca7
--- /dev/null
+++ b/test.py
@@ -0,0 +1,206 @@
+import torch
+import thop
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision
+from flaml import tune
+from collections import defaultdict
+import math
+import numpy as np
+
+DEVICE = torch.device("cpu")
+BATCHSIZE = 128
+N_TRAIN_EXAMPLES = BATCHSIZE * 30
+N_VALID_EXAMPLES = BATCHSIZE * 10
+
+
+def _BraninCurrin(config):
+    # Rescale brain
+    x_1 = 15 * config["x1"] - 5
+    x_2 = 15 * config["x2"]
+    # Brain function
+    t1 = x_2 - 5.1 / (4 * math.pi**2) * x_1**2 + 5 / math.pi * x_1 - 6
+    t2 = 10 * (1 - 1 / (8 * math.pi)) * math.cos(x_1)
+    brain_result = t1**2 + t2 + 10
+    # Currin function
+    xc_1 = config["x1"]
+    xc_2 = config["x2"]
+    factor1 = 1 - math.exp(-1 / (2 * xc_2))
+    numer = 2300 * pow(xc_1, 3) + 1900 * pow(xc_1, 2) + 2092 * xc_1 + 60
+    denom = 100 * pow(xc_1, 3) + 500 * pow(xc_1, 2) + 4 * xc_1 + 20
+    currin_result = factor1 * numer / denom
+    return {"brain": brain_result, "currin": currin_result}
+
+
+def test_lexiflow():
+    train_dataset = torchvision.datasets.FashionMNIST(
+        "test/data",
+        train=True,
+        download=True,
+        transform=torchvision.transforms.ToTensor(),
+    )
+
+    train_loader = torch.utils.data.DataLoader(
+        torch.utils.data.Subset(train_dataset, list(range(N_TRAIN_EXAMPLES))),
+        batch_size=BATCHSIZE,
+        shuffle=True,
+    )
+
+    val_dataset = torchvision.datasets.FashionMNIST(
+        "test/data", train=False, transform=torchvision.transforms.ToTensor()
+    )
+
+    val_loader = torch.utils.data.DataLoader(
+        torch.utils.data.Subset(val_dataset, list(range(N_VALID_EXAMPLES))),
+        batch_size=BATCHSIZE,
+        shuffle=True,
+    )
+
+    def define_model(configuration):
+        n_layers = configuration["n_layers"]
+        layers = []
+        in_features = 28 * 28
+        for i in range(n_layers):
+            out_features = configuration["n_units_l{}".format(i)]
+            layers.append(nn.Linear(in_features, out_features))
+            layers.append(nn.ReLU())
+            p = configuration["dropout_{}".format(i)]
+            layers.append(nn.Dropout(p))
+            in_features = out_features
+        layers.append(nn.Linear(in_features, 10))
+        layers.append(nn.LogSoftmax(dim=1))
+        return nn.Sequential(*layers)
+
+    def train_model(model, optimizer, train_loader):
+        model.train()
+        for batch_idx, (data, target) in enumerate(train_loader):
+            data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)
+            optimizer.zero_grad()
+            F.nll_loss(model(data), target).backward()
+            optimizer.step()
+
+    def eval_model(model, valid_loader):
+        model.eval()
+        correct = 0
+        with torch.no_grad():
+            for batch_idx, (data, target) in enumerate(valid_loader):
+                data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)
+                pred = model(data).argmax(dim=1, keepdim=True)
+                correct += pred.eq(target.view_as(pred)).sum().item()
+
+        accuracy = correct / N_VALID_EXAMPLES
+        flops, params = thop.profile(model, inputs=(torch.randn(1, 28 * 28).to(DEVICE),), verbose=False)
+        return np.log2(flops), 1 - accuracy, params
+
+    def evaluate_function(configuration):
+        model = define_model(configuration).to(DEVICE)
+        optimizer = torch.optim.Adam(model.parameters(), configuration["lr"])
+        n_epoch = configuration["n_epoch"]
+        for epoch in range(n_epoch):
+            train_model(model, optimizer, train_loader)
+        flops, error_rate, params = eval_model(model, val_loader)
+        return {"error_rate": error_rate, "flops": flops, "params": params}
+
+    lexico_objectives = {}
+    lexico_objectives["metrics"] = ["error_rate", "flops"]
+
+    search_space = {
+        "n_layers": tune.randint(lower=1, upper=3),
+        "n_units_l0": tune.randint(lower=4, upper=128),
+        "n_units_l1": tune.randint(lower=4, upper=128),
+        "n_units_l2": tune.randint(lower=4, upper=128),
+        "dropout_0": tune.uniform(lower=0.2, upper=0.5),
+        "dropout_1": tune.uniform(lower=0.2, upper=0.5),
+        "dropout_2": tune.uniform(lower=0.2, upper=0.5),
+        "lr": tune.loguniform(lower=1e-5, upper=1e-1),
+        "n_epoch": tune.randint(lower=1, upper=20),
+    }
+
+    low_cost_partial_config = {
+        "n_layers": 1,
+        "n_units_l0": 4,
+        "n_units_l1": 4,
+        "n_units_l2": 4,
+        "n_epoch": 1,
+    }
+
+    # Non lexico tune
+    # analysis = tune.run(
+    #     evaluate_function,
+    #     metric="error_rate",
+    #     mode="min",
+    #     num_samples=5,
+    #     config=search_space,
+    #     use_ray=False,
+    #     lexico_objectives=None,
+    #     low_cost_partial_config=low_cost_partial_config,
+    # )
+    # print(analysis.best_trial)
+    # print(analysis.best_config)
+    # print(analysis.best_result)
+
+    # # lexico tune
+    lexico_objectives["targets"] = {"error_rate": 0.0, "flops": 0.0}
+    lexico_objectives["modes"] = ["min", "min"]
+    lexico_objectives["lexico_algorithm"] = "blendsearch"
+
+    # # 1. lexico tune: absolute tolerance
+    # lexico_objectives["tolerances"] = {"error_rate": 0.02, "flops": 0.0}
+    # analysis = tune.run(
+    #     evaluate_function,
+    #     num_samples=5,
+    #     config=search_space,
+    #     use_ray=False,
+    #     lexico_objectives=lexico_objectives,
+    #     low_cost_partial_config=low_cost_partial_config,
+    # )
+    # print(analysis.best_trial)
+    # print(analysis.best_config)
+    # print(analysis.best_result)
+
+    # # 2. lexico tune: percentage tolerance
+    lexico_objectives["tolerances"] = {"error_rate": "10%", "flops": "0%"}
+    analysis = tune.run(
+        evaluate_function,
+        num_samples=5,
+        config=search_space,
+        use_ray=False,
+        lexico_objectives=lexico_objectives,
+        low_cost_partial_config=low_cost_partial_config,
+    )
+    print(analysis.best_trial)
+    print(analysis.best_config)
+    print(analysis.best_result)
+
+
+def test_lexiflow_performance():
+    lexico_objectives = {}
+    lexico_objectives["metrics"] = ["brain", "currin"]
+    lexico_objectives["tolerances"] = {"brain": 10.0, "currin": 0.0}
+    lexico_objectives["targets"] = {"brain": 0.0, "currin": 0.0}
+    lexico_objectives["modes"] = ["min", "min"]
+    lexico_objectives["lexico_algorithm"] = "blendsearch"
+
+    search_space = {
+        "x1": tune.uniform(lower=0.000001, upper=1.0),
+        "x2": tune.uniform(lower=0.000001, upper=1.0),
+    }
+
+    analysis = tune.run(
+        _BraninCurrin,
+        num_samples=1000,
+        config=search_space,
+        use_ray=False,
+        lexico_objectives=lexico_objectives,
+    )
+
+    print(analysis.best_trial)
+    print(analysis.best_config)
+    print(analysis.best_result)
+
+    assert analysis.best_result["currin"] <= 2.2, "the value of currin function should be less than 2.2"
+
+
+if __name__ == "__main__":
+    test_lexiflow()
+    # test_lexiflow_performance()

From 142bf167913b909f993cbeed9490d2e7187ea974 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Thu, 15 Jun 2023 15:26:27 +0800
Subject: [PATCH 05/15] update

---
 test.py | 206 --------------------------------------------------------
 1 file changed, 206 deletions(-)
 delete mode 100644 test.py

diff --git a/test.py b/test.py
deleted file mode 100644
index 04ae488ca7..0000000000
--- a/test.py
+++ /dev/null
@@ -1,206 +0,0 @@
-import torch
-import thop
-import torch.nn as nn
-import torch.nn.functional as F
-import torchvision
-from flaml import tune
-from collections import defaultdict
-import math
-import numpy as np
-
-DEVICE = torch.device("cpu")
-BATCHSIZE = 128
-N_TRAIN_EXAMPLES = BATCHSIZE * 30
-N_VALID_EXAMPLES = BATCHSIZE * 10
-
-
-def _BraninCurrin(config):
-    # Rescale brain
-    x_1 = 15 * config["x1"] - 5
-    x_2 = 15 * config["x2"]
-    # Brain function
-    t1 = x_2 - 5.1 / (4 * math.pi**2) * x_1**2 + 5 / math.pi * x_1 - 6
-    t2 = 10 * (1 - 1 / (8 * math.pi)) * math.cos(x_1)
-    brain_result = t1**2 + t2 + 10
-    # Currin function
-    xc_1 = config["x1"]
-    xc_2 = config["x2"]
-    factor1 = 1 - math.exp(-1 / (2 * xc_2))
-    numer = 2300 * pow(xc_1, 3) + 1900 * pow(xc_1, 2) + 2092 * xc_1 + 60
-    denom = 100 * pow(xc_1, 3) + 500 * pow(xc_1, 2) + 4 * xc_1 + 20
-    currin_result = factor1 * numer / denom
-    return {"brain": brain_result, "currin": currin_result}
-
-
-def test_lexiflow():
-    train_dataset = torchvision.datasets.FashionMNIST(
-        "test/data",
-        train=True,
-        download=True,
-        transform=torchvision.transforms.ToTensor(),
-    )
-
-    train_loader = torch.utils.data.DataLoader(
-        torch.utils.data.Subset(train_dataset, list(range(N_TRAIN_EXAMPLES))),
-        batch_size=BATCHSIZE,
-        shuffle=True,
-    )
-
-    val_dataset = torchvision.datasets.FashionMNIST(
-        "test/data", train=False, transform=torchvision.transforms.ToTensor()
-    )
-
-    val_loader = torch.utils.data.DataLoader(
-        torch.utils.data.Subset(val_dataset, list(range(N_VALID_EXAMPLES))),
-        batch_size=BATCHSIZE,
-        shuffle=True,
-    )
-
-    def define_model(configuration):
-        n_layers = configuration["n_layers"]
-        layers = []
-        in_features = 28 * 28
-        for i in range(n_layers):
-            out_features = configuration["n_units_l{}".format(i)]
-            layers.append(nn.Linear(in_features, out_features))
-            layers.append(nn.ReLU())
-            p = configuration["dropout_{}".format(i)]
-            layers.append(nn.Dropout(p))
-            in_features = out_features
-        layers.append(nn.Linear(in_features, 10))
-        layers.append(nn.LogSoftmax(dim=1))
-        return nn.Sequential(*layers)
-
-    def train_model(model, optimizer, train_loader):
-        model.train()
-        for batch_idx, (data, target) in enumerate(train_loader):
-            data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)
-            optimizer.zero_grad()
-            F.nll_loss(model(data), target).backward()
-            optimizer.step()
-
-    def eval_model(model, valid_loader):
-        model.eval()
-        correct = 0
-        with torch.no_grad():
-            for batch_idx, (data, target) in enumerate(valid_loader):
-                data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)
-                pred = model(data).argmax(dim=1, keepdim=True)
-                correct += pred.eq(target.view_as(pred)).sum().item()
-
-        accuracy = correct / N_VALID_EXAMPLES
-        flops, params = thop.profile(model, inputs=(torch.randn(1, 28 * 28).to(DEVICE),), verbose=False)
-        return np.log2(flops), 1 - accuracy, params
-
-    def evaluate_function(configuration):
-        model = define_model(configuration).to(DEVICE)
-        optimizer = torch.optim.Adam(model.parameters(), configuration["lr"])
-        n_epoch = configuration["n_epoch"]
-        for epoch in range(n_epoch):
-            train_model(model, optimizer, train_loader)
-        flops, error_rate, params = eval_model(model, val_loader)
-        return {"error_rate": error_rate, "flops": flops, "params": params}
-
-    lexico_objectives = {}
-    lexico_objectives["metrics"] = ["error_rate", "flops"]
-
-    search_space = {
-        "n_layers": tune.randint(lower=1, upper=3),
-        "n_units_l0": tune.randint(lower=4, upper=128),
-        "n_units_l1": tune.randint(lower=4, upper=128),
-        "n_units_l2": tune.randint(lower=4, upper=128),
-        "dropout_0": tune.uniform(lower=0.2, upper=0.5),
-        "dropout_1": tune.uniform(lower=0.2, upper=0.5),
-        "dropout_2": tune.uniform(lower=0.2, upper=0.5),
-        "lr": tune.loguniform(lower=1e-5, upper=1e-1),
-        "n_epoch": tune.randint(lower=1, upper=20),
-    }
-
-    low_cost_partial_config = {
-        "n_layers": 1,
-        "n_units_l0": 4,
-        "n_units_l1": 4,
-        "n_units_l2": 4,
-        "n_epoch": 1,
-    }
-
-    # Non lexico tune
-    # analysis = tune.run(
-    #     evaluate_function,
-    #     metric="error_rate",
-    #     mode="min",
-    #     num_samples=5,
-    #     config=search_space,
-    #     use_ray=False,
-    #     lexico_objectives=None,
-    #     low_cost_partial_config=low_cost_partial_config,
-    # )
-    # print(analysis.best_trial)
-    # print(analysis.best_config)
-    # print(analysis.best_result)
-
-    # # lexico tune
-    lexico_objectives["targets"] = {"error_rate": 0.0, "flops": 0.0}
-    lexico_objectives["modes"] = ["min", "min"]
-    lexico_objectives["lexico_algorithm"] = "blendsearch"
-
-    # # 1. lexico tune: absolute tolerance
-    # lexico_objectives["tolerances"] = {"error_rate": 0.02, "flops": 0.0}
-    # analysis = tune.run(
-    #     evaluate_function,
-    #     num_samples=5,
-    #     config=search_space,
-    #     use_ray=False,
-    #     lexico_objectives=lexico_objectives,
-    #     low_cost_partial_config=low_cost_partial_config,
-    # )
-    # print(analysis.best_trial)
-    # print(analysis.best_config)
-    # print(analysis.best_result)
-
-    # # 2. lexico tune: percentage tolerance
-    lexico_objectives["tolerances"] = {"error_rate": "10%", "flops": "0%"}
-    analysis = tune.run(
-        evaluate_function,
-        num_samples=5,
-        config=search_space,
-        use_ray=False,
-        lexico_objectives=lexico_objectives,
-        low_cost_partial_config=low_cost_partial_config,
-    )
-    print(analysis.best_trial)
-    print(analysis.best_config)
-    print(analysis.best_result)
-
-
-def test_lexiflow_performance():
-    lexico_objectives = {}
-    lexico_objectives["metrics"] = ["brain", "currin"]
-    lexico_objectives["tolerances"] = {"brain": 10.0, "currin": 0.0}
-    lexico_objectives["targets"] = {"brain": 0.0, "currin": 0.0}
-    lexico_objectives["modes"] = ["min", "min"]
-    lexico_objectives["lexico_algorithm"] = "blendsearch"
-
-    search_space = {
-        "x1": tune.uniform(lower=0.000001, upper=1.0),
-        "x2": tune.uniform(lower=0.000001, upper=1.0),
-    }
-
-    analysis = tune.run(
-        _BraninCurrin,
-        num_samples=1000,
-        config=search_space,
-        use_ray=False,
-        lexico_objectives=lexico_objectives,
-    )
-
-    print(analysis.best_trial)
-    print(analysis.best_config)
-    print(analysis.best_result)
-
-    assert analysis.best_result["currin"] <= 2.2, "the value of currin function should be less than 2.2"
-
-
-if __name__ == "__main__":
-    test_lexiflow()
-    # test_lexiflow_performance()

From cfbf7d32f784fee7325abdadaec01a3b5a57cba0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Thu, 15 Jun 2023 20:49:01 +0800
Subject: [PATCH 06/15] update

---
 flaml/tune/searcher/blendsearch.py   |  33 +++------
 flaml/tune/searcher/flow2.py         |  16 ++--
 flaml/tune/searcher/search_thread.py | 103 ++++++++++++++------------
 flaml/tune/searcher/suggestion.py    | 105 +++++++++++++++++++++++++++
 flaml/tune/tune.py                   |   6 +-
 5 files changed, 184 insertions(+), 79 deletions(-)

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index b90421c0fa..cc3d5ea6f8 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -21,6 +21,7 @@
 except (ImportError, AssertionError):
     from .suggestion import Searcher
     from .suggestion import OptunaSearch as GlobalSearch
+from .suggestion import LexiGlobalSearch as LexiGlobalSearch
 from ..trial import unflatten_dict, flatten_dict
 from .. import INCUMBENT_RESULT
 from .search_thread import SearchThread
@@ -35,22 +36,6 @@
 logger = logging.getLogger(__name__)
 
 
-class Lexico_GlobalSearch(GlobalSearch):
-    def set_search_properties(
-        self, metric: Optional[Union[str, List[str]]], mode: Optional[Union[str, List[str]]], config: Dict
-    ) -> bool:
-        if self._space:
-            return False
-        space = self.convert_search_space(config)
-        self._space = space
-        if metric:
-            self._metric = metric
-        if mode:
-            self._mode = mode
-        self._setup_study(mode)
-        return True
-
-
 class BlendSearch(Searcher):
     """class for BlendSearch algorithm."""
 
@@ -276,7 +261,7 @@ def __init__(
             try:
                 assert evaluated_rewards
                 if self.lexico_objectives:
-                    self._gs = Lexico_GlobalSearch(
+                    self._gs = LexiGlobalSearch(
                         space=gs_space,
                         metric=metric,
                         mode=mode,
@@ -285,7 +270,7 @@ def __init__(
                         points_to_evaluate=self._evaluated_points,
                         evaluated_rewards=evaluated_rewards,
                     )
-                    Lexico_GlobalSearch.lexico_objectives = self.lexico_objective
+                    LexiGlobalSearch.lexico_objectives = self.lexico_objective
                 else:
                     self._gs = GlobalSearch(
                         space=gs_space,
@@ -298,14 +283,14 @@ def __init__(
                     )
             except (AssertionError, ValueError):
                 if self.lexico_objectives:
-                    self._gs = Lexico_GlobalSearch(
+                    self._gs = LexiGlobalSearch(
                         space=gs_space,
                         metric=metric,
                         mode=mode,
                         seed=gs_seed,
                         sampler=sampler,
                     )
-                    Lexico_GlobalSearch.lexico_objectives = self.lexico_objectives
+                    LexiGlobalSearch.lexico_objectives = self.lexico_objectives
                 else:
                     self._gs = GlobalSearch(
                         space=gs_space,
@@ -391,7 +376,7 @@ def set_search_properties(
                 self._ls.set_search_properties(metric, mode)
                 if self._gs is not None:
                     if self.lexico_objectives:
-                        self._gs = Lexico_GlobalSearch(
+                        self._gs = LexiGlobalSearch(
                             space=self._gs._space,
                             metric=metric,
                             mode=mode,
@@ -560,7 +545,7 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
                 if self._histories is None:
                     self._histories, self._f_best = defaultdict(list), {}
                 for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                    self._histories[k_metric].append(result[k_metric] if k_mode == "min" else -result[k_metric])
+                    self._histories[k_metric].append(result[k_metric] if k_mode == "min" else -1 * result[k_metric])
                 self.update_fbest()
             config = result.get("config", {})
             if not config:
@@ -838,7 +823,7 @@ def on_trial_result(self, trial_id: str, result: Dict):
                 if self._histories is None:
                     self._histories, self._f_best = defaultdict(list), {}
                 for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                    self._histories[k_metric].append(result[k_metric] if k_mode == "min" else -result[k_metric])
+                    self._histories[k_metric].append(result[k_metric] if k_mode == "min" else -1 * result[k_metric])
                 self.update_fbest()
         self._search_thread_pool[thread_id].on_trial_result(trial_id, result)
 
@@ -1255,7 +1240,7 @@ def update_search_space(self, search_space):
         )
         if self._gs is not None:
             if self.lexico_objectives:
-                self._gs = Lexico_GlobalSearch(
+                self._gs = LexiGlobalSearch(
                     space=config,
                     metric=self._metric,
                     mode=self._mode,
diff --git a/flaml/tune/searcher/flow2.py b/flaml/tune/searcher/flow2.py
index 6b75a3edaf..09cebce672 100644
--- a/flaml/tune/searcher/flow2.py
+++ b/flaml/tune/searcher/flow2.py
@@ -134,7 +134,7 @@ def __init__(
         self.max_resource = max_resource
         self._resource = None
         self._f_best = None  # only use for lexico_comapre. It represent the best value achieved by lexico_flow.
-        self.op_dimension = None 
+        self.op_dimension = None
         self._step_lb = np.Inf
         self._histories = None  # only use for lexico_comapre. It records the result of historical configurations.
         if space is not None:
@@ -184,7 +184,7 @@ def _init_search(self):
         self.incumbent = {}
         self.incumbent = self.normalize(self.best_config)  # flattened
         self.best_obj = self.cost_incumbent = None
-        self.pre_best_obj = None
+        # self.pre_best_obj = None
         self.dim = len(self._tunable_keys)  # total # tunable dimensions
         self._direction_tried = None
         self._num_complete4incumbent = self._cost_complete4incumbent = 0
@@ -296,7 +296,9 @@ def create(self, init_config: Dict, obj: float, cost: float, space: Dict) -> Sea
             flow2.best_obj = {}
             for k, v in obj.items():
                 flow2.best_obj[k] = (
-                    -1 * v if self.lexico_objectives["modes"][self.lexico_objectives["metrics"].index(k)] == "max" else v
+                    -1 * v
+                    if self.lexico_objectives["modes"][self.lexico_objectives["metrics"].index(k)] == "max"
+                    else v
                 )
         else:
             flow2.best_obj = obj * self.metric_op  # minimize internally
@@ -375,7 +377,7 @@ def lexico_compare(self, result) -> bool:
                 k_target = (
                     self.lexico_objectives["targets"][k_metric]
                     if k_mode == "min"
-                    else -self.lexico_objectives["targets"][k_metric]
+                    else -1 * self.lexico_objectives["targets"][k_metric]
                 )
                 if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
                     tolerance_bound = self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
@@ -438,7 +440,7 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
                     or (self.lexico_objectives is None and obj < self.best_obj)
                     or (self.lexico_objectives is not None and self.lexico_compare(obj))
                 ):
-                    self.pre_best_obj = self.best_obj
+                    # self.pre_best_obj = self.best_obj
                     self.best_obj = obj
                     self.best_config, self.step = self._configs[trial_id]
                     self.incumbent = self.normalize(self.best_config)
@@ -496,7 +498,7 @@ def on_trial_result(self, trial_id: str, result: Dict):
                     or (self.lexico_objectives is None and obj < self.best_obj)
                     or (self.lexico_objectives is not None and self.lexico_compare(obj))
                 ):
-                    self.pre_best_obj = self.best_obj
+                    # self.pre_best_obj = self.best_obj
                     self.best_obj = obj
                     config = self._configs[trial_id][0]
                     if self.best_config != config:
@@ -674,4 +676,4 @@ def reach(self, other: Searcher) -> bool:
             if config1[key] != config2.get(key):
                 return False
         delta = np.array([incumbent1[key] - incumbent2.get(key, np.inf) for key in self._tunable_keys])
-        return np.linalg.norm(delta) <= self.step
\ No newline at end of file
+        return np.linalg.norm(delta) <= self.step
diff --git a/flaml/tune/searcher/search_thread.py b/flaml/tune/searcher/search_thread.py
index ea8b79e789..3944c5411e 100644
--- a/flaml/tune/searcher/search_thread.py
+++ b/flaml/tune/searcher/search_thread.py
@@ -2,7 +2,7 @@
 #  * Copyright (c) Microsoft Corporation. All rights reserved.
 #  * Licensed under the MIT License. See LICENSE file in the
 #  * project root for license information.
-from typing import Dict, Optional
+from typing import Dict, Optional, Union
 import numpy as np
 
 try:
@@ -46,9 +46,9 @@ def __init__(
         if self.lexico_objectives:
             # set 1st to 0 others to -1?
             self.obj_best1 = self.obj_best2 = {}
-            for k in self.lexico_objectives["metrics"]:
-                self.obj_best1[k] = self.obj_best2[k] = (
-                    np.inf if getattr(search_alg, "best_obj", None) is None else search_alg.best_obj[k]
+            for k_metric in self.lexico_objectives["metrics"]:
+                self.obj_best1[k_metric] = self.obj_best2[k_metric] = (
+                    np.inf if getattr(search_alg, "best_obj", None) is None else search_alg.best_obj[k_metric]
                 )
         else:
             self.obj_best1 = self.obj_best2 = getattr(search_alg, "best_obj", np.inf)  # inherently minimize
@@ -90,23 +90,8 @@ def suggest(self, trial_id: str) -> Optional[Dict]:
             self.running += 1
         return config
 
-    def _get_lexico_bound(self, metric, mode):
-        k_target = (
-            self.lexico_objectives["targets"][metric] if mode == "min" else -self.lexico_objectives["targets"][metric]
-        )
-        if not isinstance(self.lexico_objectives["tolerances"][metric], str):
-            tolerance_bound = self._f_best[metric] + self.lexico_objectives["tolerances"][metric]
-        else:
-            assert (
-                self.lexico_objectives["tolerances"][metric][-1] == "%"
-            ), "String tolerance of {} should use %% as the suffix".format(metric)
-            tolerance_bound = self._f_best[metric] * (
-                1 + 0.01 * float(self.lexico_objectives["tolerances"][metric].replace("%", ""))
-            )
-        bound = max(tolerance_bound, k_target)
-        return bound
-
     def update_priority(self, eci: Optional[float] = 0):
+        # optimistic projection
         if self.lexico_objectives:
             for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
                 self.priority[k_metric] = eci * self.speed[k_metric] - self.obj_best1[k_metric]
@@ -115,7 +100,6 @@ def update_priority(self, eci: Optional[float] = 0):
 
     def update_eci(self, metric_target: float, max_speed: Optional[float] = np.inf, min_speed: Optional[float] = 1e-9):
         # calculate eci: estimated cost for improvement over metric_target
-        # if lexico, metric_target = _f_best[_metric_1st], else global best
         if self.lexico_objectives is None:
             _metric_op = self._metric_op
             if not self.speed:
@@ -129,35 +113,64 @@ def update_eci(self, metric_target: float, max_speed: Optional[float] = np.inf,
                 self.speed = min_speed
         best_obj = metric_target * _metric_op
         self.eci = max(self.cost_total - self.cost_best1, self.cost_best1 - self.cost_best2)
-        # get "obj_best1" and "speed"
         obj_best1 = self.obj_best1 if not self.lexico_objectives else self.obj_best1[_metric_1st]
         speed = self.speed if not self.lexico_objectives else self.speed[_metric_1st]
         if obj_best1 > best_obj and speed > 0:
             self.eci = max(self.eci, 2 * (self.obj_best1 - best_obj) / self.speed)
 
+    def _better(self, obj_1: Union[dict, float], obj_2: Union[dict, float]) -> bool:
+        if self.lexico_objectives:
+            for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
+                bound = self._search_alg._get_lexico_bound(k_metric, k_mode)
+                if (obj_1[k_metric] < bound) and (obj_2[k_metric] < bound):
+                    continue
+                elif obj_1[k_metric] < obj_2[k_metric]:
+                    return (True, k_metric)
+                else:
+                    return (False, None)
+            for k_metr in self.lexico_objectives["metrics"]:
+                if obj_1[k_metr] == obj_2[k_metr]:
+                    continue
+                elif obj_1[k_metr] < obj_2[k_metr]:
+                    return (True, k_metric)
+                else:
+                    return (False, None)
+        else:
+            if obj_1 < obj_2:
+                return True
+            else:
+                return False
+
     def _update_speed(self):
         # calculate speed; use 0 for invalid speed temporarily
-        if self.lexico_objectives is None and self.obj_best2 > self.obj_best1:
-            self.speed = (
-                (self.obj_best2 - self.obj_best1) / self.running / (max(self.cost_total - self.cost_best2, self._eps))
-            )
-        elif self.lexico_objectives is not None and self.obj_best2 != self.obj_best1:
-            op_dimension = self._search_alg.op_dimension
-            op_index = self.lexico_objectives["metrics"].index(op_dimension)
-            metrics_length = len(self.lexico_objectives["metrics"])
-            self.speed[op_dimension] = (
-                (self.obj_best2[op_dimension] - self.obj_best1[op_dimension])
-                / self.running
-                / (max(self.cost_total - self.cost_best2, self._eps))
-            )
-            for i in range(0, metrics_length):
-                if i < op_index:
-                    self.speed[self.lexico_objectives["metrics"][i]] = -1
-                elif i > op_index:
-                    self.speed[self.lexico_objectives["metrics"][i]] = 0
-        else:
-            if self.lexico_objectives is None:
+        if self.lexico_objectives is None:
+            if self._better(self.obj_best1, self.obj_best2):
+                self.speed = (
+                    (self.obj_best2 - self.obj_best1)
+                    / self.running
+                    / (max(self.cost_total - self.cost_best2, self._eps))
+                )
+            else:
                 self.speed = 0
+        else:
+            compare_tuple = self._better(self.obj_best1, self.obj_best2)
+            if compare_tuple[0]:
+                if self._is_ls:
+                    op_dimension = self._search_alg.op_dimension
+                else:
+                    op_dimension = self._better(self.obj_best1, self.obj_best2)[1]
+                op_index = self.lexico_objectives["metrics"].index(op_dimension)
+                metrics_length = len(self.lexico_objectives["metrics"])
+                self.speed[op_dimension] = (
+                    (self.obj_best2[op_dimension] - self.obj_best1[op_dimension])
+                    / self.running
+                    / (max(self.cost_total - self.cost_best2, self._eps))
+                )
+                for i in range(0, metrics_length):
+                    if i < op_index:
+                        self.speed[self.lexico_objectives["metrics"][i]] = -1
+                    elif i > op_index:
+                        self.speed[self.lexico_objectives["metrics"][i]] = 0
             else:
                 for i in range(0, len(self.lexico_objectives["metrics"])):
                     self.speed[self.lexico_objectives["metrics"][i]] = 0
@@ -195,11 +208,7 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
                         self._search_alg.lexico_objectives["metrics"], self._search_alg.lexico_objectives["modes"]
                     ):
                         obj[k] = -result[k] if m == "max" else result[k]
-                if (
-                    self.best_result is None
-                    or (self.lexico_objectives is None and obj < self.obj_best1)
-                    or (self.lexico_objectives is not None and obj == self._search_alg.best_obj)
-                ):
+                if self.best_result is None or self._better(obj, self.obj_best1):
                     self.cost_best2 = self.cost_best1
                     self.cost_best1 = self.cost_total
                     if self.lexico_objectives is None:
diff --git a/flaml/tune/searcher/suggestion.py b/flaml/tune/searcher/suggestion.py
index 518bab9da9..fc1b92c153 100644
--- a/flaml/tune/searcher/suggestion.py
+++ b/flaml/tune/searcher/suggestion.py
@@ -20,6 +20,7 @@
 import warnings
 import copy
 import logging
+import numpy as np
 from typing import Any, Dict, Optional, Union, List, Tuple, Callable
 import pickle
 from .variant_generator import parse_spec_vars
@@ -34,6 +35,13 @@
 )
 from ..trial import flatten_dict, unflatten_dict
 
+from ray import __version__ as ray_version
+
+assert ray_version >= "1.10.0"
+if ray_version.startswith("1."):
+    from ray.tune.suggest.optuna import OptunaSearch as MOSearch
+else:
+    from ray.tune.search.optuna import OptunaSearch as MOSearch
 logger = logging.getLogger(__name__)
 
 UNRESOLVED_SEARCH_SPACE = str(
@@ -739,3 +747,100 @@ def resolve_value(domain: Domain) -> ot.distributions.BaseDistribution:
         values = {"/".join(path): resolve_value(domain) for path, domain in domain_vars}
 
         return values
+
+
+class LexiGlobalSearch(MOSearch):
+    def __init__(
+        self,
+        space: Optional[
+            Union[
+                Dict[str, "OptunaDistribution"],
+                List[Tuple],
+                Callable[["OptunaTrial"], Optional[Dict[str, Any]]],
+            ]
+        ] = None,
+        metric: Optional[str] = None,
+        mode: Optional[str] = None,
+        points_to_evaluate: Optional[List[Dict]] = None,
+        sampler: Optional["BaseSampler"] = None,
+        seed: Optional[int] = None,
+        evaluated_rewards: Optional[List] = None,
+    ):
+        super().__init__(space, metric, mode, points_to_evaluate, sampler, seed, evaluated_rewards)
+        self._f_best = None  # only use for lexico_comapre. It represent the best value achieved by the algorithm.
+        self._histories = None  # only use for lexico_comapre. It records the result of historical configurations.
+
+    def set_search_properties(
+        self, metric: Optional[Union[str, List[str]]], mode: Optional[Union[str, List[str]]], config: Dict
+    ) -> bool:
+        if self._space:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+        self._setup_study(mode)
+        return True
+
+    def update_fbest(
+        self,
+    ):
+        obj_initial = self.lexico_objectives["metrics"][0]
+        feasible_index = np.array([*range(len(self._histories[obj_initial]))])
+        for k_metric in self.lexico_objectives["metrics"]:
+            k_values = np.array(self._histories[k_metric])
+            feasible_value = k_values.take(feasible_index)
+            self._f_best[k_metric] = np.min(feasible_value)
+            if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
+                tolerance_bound = self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
+            else:
+                assert (
+                    self.lexico_objectives["tolerances"][k_metric][-1] == "%"
+                ), "String tolerance of {} should use %% as the suffix".format(k_metric)
+                tolerance_bound = self._f_best[k_metric] * (
+                    1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))
+                )
+            feasible_index_filter = np.where(
+                feasible_value
+                <= max(
+                    tolerance_bound,
+                    self.lexico_objectives["targets"][k_metric],
+                )
+            )[0]
+            feasible_index = feasible_index.take(feasible_index_filter)
+
+    def _get_lexico_bound(self, metric, mode):
+        k_target = (
+            self.lexico_objectives["targets"][metric]
+            if mode == "min"
+            else -1 * self.lexico_objectives["targets"][metric]
+        )
+        if not isinstance(self.lexico_objectives["tolerances"][metric], str):
+            tolerance_bound = self._f_best[metric] + self.lexico_objectives["tolerances"][metric]
+        else:
+            assert (
+                self.lexico_objectives["tolerances"][metric][-1] == "%"
+            ), "String tolerance of {} should use %% as the suffix".format(metric)
+            tolerance_bound = self._f_best[metric] * (
+                1 + 0.01 * float(self.lexico_objectives["tolerances"][metric].replace("%", ""))
+            )
+        bound = max(tolerance_bound, k_target)
+        return bound
+
+    def on_trial_result(self, trial_id: str, result: Dict):
+        super.on_trial_result(trial_id, result)
+        for k_metric, k_mode in zip(self.lexico_objectives["metrics"]):
+            self._histories[k_metric].append(result[k_metric]) if k_mode == "min" else self._histories[k_metric].append(
+                result[k_metric] * -1
+            )
+        self.update_fbest()
+
+    def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
+        super.on_trial_complete(trial_id, result, error)
+        for k_metric, k_mode in zip(self.lexico_objectives["metrics"]):
+            self._histories[k_metric].append(result[k_metric]) if k_mode == "min" else self._histories[k_metric].append(
+                result[k_metric] * -1
+            )
+        self.update_fbest()
diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py
index 031d828624..52a0c47eb7 100644
--- a/flaml/tune/tune.py
+++ b/flaml/tune/tune.py
@@ -82,7 +82,7 @@ def lexico_best(self, trials):
         for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
             k_values = np.array(histories[k_metric])
             k_target = (
-                -self.lexico_objectives["targets"][k_metric]
+                -1 * self.lexico_objectives["targets"][k_metric]
                 if k_mode == "max"
                 else self.lexico_objectives["targets"][k_metric]
             )
@@ -492,6 +492,10 @@ def easy_objective(config):
                 logger.warning("Using CFO for search. To use BlendSearch, run: pip install flaml[blendsearch]")
             metric = metric or DEFAULT_METRIC
         else:
+            assert "lexico_algorithm" in lexico_objectives and lexico_objectives["lexico_algorithm"] in [
+                "CFO",
+                "BlendSearch",
+            ], 'When performing lexicographic optimization, "lexico_algorithm" should be provided in lexicographic objectives (CFO or BlendSearch).'
             SearchAlgorithm = CFO if lexico_objectives["lexico_algorithm"] == "CFO" else BlendSearch
             logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
             metric = lexico_objectives["metrics"][0] or DEFAULT_METRIC

From 3f220cda82da9c97cf9669cf3a1e42bd3f481f58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Fri, 16 Jun 2023 15:25:56 +0800
Subject: [PATCH 07/15] update

---
 flaml/tune/searcher/blendsearch.py   | 28 ++++++++--------
 flaml/tune/searcher/flow2.py         | 49 ++++++++++++++--------------
 flaml/tune/searcher/search_thread.py | 22 ++++++-------
 flaml/tune/tune.py                   |  6 ++--
 4 files changed, 52 insertions(+), 53 deletions(-)

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index cc3d5ea6f8..eec85097ac 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -169,7 +169,6 @@ def __init__(
             self._metric = self.lexico_objectives["metrics"][0]
             self._mode = self.lexico_objectives["modes"][0]
         self._use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
-        self.lexico_objectives = lexico_objectives
         self._histories = None
         self._f_best = None
         init_config = low_cost_partial_config or {}
@@ -254,10 +253,7 @@ def __init__(
             else:
                 sampler = None
             if self.lexico_objectives:
-                metric, mode = [], []
-                for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                    metric.append(k_metric)
-                    mode.append(k_mode)
+                metric, mode = self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
             try:
                 assert evaluated_rewards
                 if self.lexico_objectives:
@@ -270,7 +266,6 @@ def __init__(
                         points_to_evaluate=self._evaluated_points,
                         evaluated_rewards=evaluated_rewards,
                     )
-                    LexiGlobalSearch.lexico_objectives = self.lexico_objective
                 else:
                     self._gs = GlobalSearch(
                         space=gs_space,
@@ -290,7 +285,6 @@ def __init__(
                         seed=gs_seed,
                         sampler=sampler,
                     )
-                    LexiGlobalSearch.lexico_objectives = self.lexico_objectives
                 else:
                     self._gs = GlobalSearch(
                         space=gs_space,
@@ -299,6 +293,8 @@ def __init__(
                         seed=gs_seed,
                         sampler=sampler,
                     )
+            if isinstance(self._gs, LexiGlobalSearch):
+                self._gs.lexico_objectives = self.lexico_objectives
             self._gs.space = space
         else:
             self._gs = None
@@ -389,6 +385,8 @@ def set_search_properties(
                             mode=mode,
                             seed=self._gs_seed,
                         )
+                    if isinstance(self._gs, LexiGlobalSearch):
+                        self._gs.lexico_objectives = self.lexico_objective
                     self._gs.space = self._ls.space
                 self._init_search()
         if spec:
@@ -401,7 +399,7 @@ def set_search_properties(
                 self._set_deadline()
                 if self._input_cost_attr == "auto" and self._time_budget_s:
                     self.cost_attr = self._ls.cost_attr = TIME_TOTAL_S
-            if "metric_target" in spec:  # works only in online_searcher
+            if "metric_target" in spec:
                 self._metric_target = spec.get("metric_target")
             num_samples = spec.get("num_samples")
             if num_samples is not None:
@@ -430,7 +428,7 @@ def _init_search(self):
         self._set_deadline()
         self._is_ls_ever_converged = False
         self._subspace = {}  # the subspace for each trial id
-        if self.lexico_objectives is None:
+        if not self.lexico_objectives:
             self._metric_target = np.inf * self._ls.metric_op
         self._search_thread_pool = {
             # id: int -> thread: SearchThread
@@ -491,7 +489,9 @@ def restore(self, checkpoint_path: str):
 
     def _get_lexico_bound(self, metric, mode):
         k_target = (
-            self.lexico_objectives["targets"][metric] if mode == "min" else -self.lexico_objectives["targets"][metric]
+            self.lexico_objectives["targets"][metric]
+            if mode == "min"
+            else -1 * self.lexico_objectives["targets"][metric]
         )
         if not isinstance(self.lexico_objectives["tolerances"][metric], str):
             tolerance_bound = self._f_best[metric] + self.lexico_objectives["tolerances"][metric]
@@ -561,7 +561,7 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
                 self._cost_used += result.get(self.cost_attr, 0)
                 self._result[signature] = result
                 # update target metric if improved
-                if self.lexico_objectives is None:
+                if not self.lexico_objectives:
                     objective = result[self._ls.metric]
                     if (objective - self._metric_target) * self._ls.metric_op < 0:
                         self._metric_target = objective
@@ -603,7 +603,7 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
             del self._subspace[trial_id]
 
     def _create_thread(self, config, result, space):
-        if self.lexico_objectives is None:
+        if not self.lexico_objectives:
             obj = result[self._ls.metric]
         else:
             obj = {k: result[k] for k in self.lexico_objectives["metrics"]}
@@ -1039,7 +1039,7 @@ def _select_thread(self) -> Tuple:
             num_proposed = num_finished + len(self._trial_proposed_by)
             min_eci = max(self._num_samples - num_proposed, 0)
         # update priority
-        if self.lexico_objectives is None:
+        if not self.lexico_objectives:
             max_speed = 0
             min_speed = float("inf")
             for thread in self._search_thread_pool.values():
@@ -1057,7 +1057,7 @@ def _select_thread(self) -> Tuple:
                     if thread.speed[k_metric] < min_speed[k_metric]:
                         min_speed[k_metric] = thread.speed[k_metric]
         for thread in self._search_thread_pool.values():
-            if self.lexico_objectives is None:
+            if not self.lexico_objectives:
                 thread.update_eci(self._metric_target, max_speed, min_speed)
             else:
                 _metric_1st = self.lexico_objectives["metrics"][0]
diff --git a/flaml/tune/searcher/flow2.py b/flaml/tune/searcher/flow2.py
index 09cebce672..4e8e4375bc 100644
--- a/flaml/tune/searcher/flow2.py
+++ b/flaml/tune/searcher/flow2.py
@@ -121,7 +121,7 @@ def __init__(
         self.resource_attr = resource_attr
         self.min_resource = min_resource
         self.lexico_objectives = lexico_objectives
-        if self.lexico_objectives is not None:
+        if self.lexico_objectives:
             if "modes" not in self.lexico_objectives.keys():
                 self.lexico_objectives["modes"] = ["min"] * len(self.lexico_objectives["metrics"])
             for t_metric, t_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
@@ -292,7 +292,7 @@ def create(self, init_config: Dict, obj: float, cost: float, space: Dict) -> Sea
             self.seed + 1,
             self.lexico_objectives,
         )
-        if self.lexico_objectives is not None:
+        if self.lexico_objectives:
             flow2.best_obj = {}
             for k, v in obj.items():
                 flow2.best_obj[k] = (
@@ -314,6 +314,24 @@ def denormalize(self, config):
         """denormalize each dimension in config from [0,1]."""
         return denormalize(config, self._space, self.best_config, self.incumbent, self._random)
 
+    def _get_lexico_bound(self, metric, mode):
+        k_target = (
+            self.lexico_objectives["targets"][metric]
+            if mode == "min"
+            else -1 * self.lexico_objectives["targets"][metric]
+        )
+        if not isinstance(self.lexico_objectives["tolerances"][metric], str):
+            tolerance_bound = self._f_best[metric] + self.lexico_objectives["tolerances"][metric]
+        else:
+            assert (
+                self.lexico_objectives["tolerances"][metric][-1] == "%"
+            ), "String tolerance of {} should use %% as the suffix".format(metric)
+            tolerance_bound = self._f_best[metric] * (
+                1 + 0.01 * float(self.lexico_objectives["tolerances"][metric].replace("%", ""))
+            )
+        bound = max(tolerance_bound, k_target)
+        return bound
+
     def set_search_properties(
         self,
         metric: Optional[str] = None,
@@ -374,27 +392,8 @@ def lexico_compare(self, result) -> bool:
                 self._histories[k].append(result[k])
             self.update_fbest()
             for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                k_target = (
-                    self.lexico_objectives["targets"][k_metric]
-                    if k_mode == "min"
-                    else -1 * self.lexico_objectives["targets"][k_metric]
-                )
-                if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
-                    tolerance_bound = self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
-                else:
-                    assert (
-                        self.lexico_objectives["tolerances"][k_metric][-1] == "%"
-                    ), "String tolerance of {} should use %% as the suffix".format(k_metric)
-                    tolerance_bound = self._f_best[k_metric] * (
-                        1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))
-                    )
-                if (result[k_metric] < max(tolerance_bound, k_target)) and (
-                    self.best_obj[k_metric]
-                    < max(
-                        tolerance_bound,
-                        k_target,
-                    )
-                ):
+                bound = self._get_lexico_bound(k_metric, k_mode)
+                if (result[k_metric] < bound) and (self.best_obj[k_metric] < bound):
                     continue
                 elif result[k_metric] < self.best_obj[k_metric]:
                     self.op_dimension = k_metric
@@ -420,7 +419,7 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
         if not error and result:
             obj = (
                 result.get(self._metric)
-                if self.lexico_objectives is None
+                if not self.lexico_objectives
                 else {k: result[k] for k in self.lexico_objectives["metrics"]}
             )
             if obj:
@@ -478,7 +477,7 @@ def on_trial_result(self, trial_id: str, result: Dict):
         if result:
             obj = (
                 result.get(self._metric)
-                if self.lexico_objectives is None
+                if not self.lexico_objectives
                 else {k: result[k] for k in self.lexico_objectives["metrics"]}
             )
             if obj:
diff --git a/flaml/tune/searcher/search_thread.py b/flaml/tune/searcher/search_thread.py
index 3944c5411e..5aa137360a 100644
--- a/flaml/tune/searcher/search_thread.py
+++ b/flaml/tune/searcher/search_thread.py
@@ -100,7 +100,7 @@ def update_priority(self, eci: Optional[float] = 0):
 
     def update_eci(self, metric_target: float, max_speed: Optional[float] = np.inf, min_speed: Optional[float] = 1e-9):
         # calculate eci: estimated cost for improvement over metric_target
-        if self.lexico_objectives is None:
+        if not self.lexico_objectives:
             _metric_op = self._metric_op
             if not self.speed:
                 self.speed = max_speed
@@ -118,7 +118,7 @@ def update_eci(self, metric_target: float, max_speed: Optional[float] = np.inf,
         if obj_best1 > best_obj and speed > 0:
             self.eci = max(self.eci, 2 * (self.obj_best1 - best_obj) / self.speed)
 
-    def _better(self, obj_1: Union[dict, float], obj_2: Union[dict, float]) -> bool:
+    def _better(self, obj_1: Union[dict, float], obj_2: Union[dict, float]):
         if self.lexico_objectives:
             for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
                 bound = self._search_alg._get_lexico_bound(k_metric, k_mode)
@@ -143,7 +143,7 @@ def _better(self, obj_1: Union[dict, float], obj_2: Union[dict, float]) -> bool:
 
     def _update_speed(self):
         # calculate speed; use 0 for invalid speed temporarily
-        if self.lexico_objectives is None:
+        if not self.lexico_objectives:
             if self._better(self.obj_best1, self.obj_best2):
                 self.speed = (
                     (self.obj_best2 - self.obj_best1)
@@ -152,13 +152,13 @@ def _update_speed(self):
                 )
             else:
                 self.speed = 0
-        else:
+        elif self._search_alg._histories:
             compare_tuple = self._better(self.obj_best1, self.obj_best2)
             if compare_tuple[0]:
                 if self._is_ls:
                     op_dimension = self._search_alg.op_dimension
                 else:
-                    op_dimension = self._better(self.obj_best1, self.obj_best2)[1]
+                    op_dimension = compare_tuple[1]
                 op_index = self.lexico_objectives["metrics"].index(op_dimension)
                 metrics_length = len(self.lexico_objectives["metrics"])
                 self.speed[op_dimension] = (
@@ -172,8 +172,8 @@ def _update_speed(self):
                     elif i > op_index:
                         self.speed[self.lexico_objectives["metrics"][i]] = 0
             else:
-                for i in range(0, len(self.lexico_objectives["metrics"])):
-                    self.speed[self.lexico_objectives["metrics"][i]] = 0
+                for k_metric in self.lexico_objectives["metrics"]:
+                    self.speed[k_metric] = 0
 
     def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
         """Update the statistics of the thread."""
@@ -195,23 +195,23 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
         if result:
             self.cost_last = result.get(self.cost_attr, 1)
             self.cost_total += self.cost_last
-            if self.lexico_objectives is None:
+            if not self.lexico_objectives:
                 feasible_condition = self._search_alg.metric in result
             else:
                 feasible_condition = all(x in result for x in self._search_alg.metric)
             if feasible_condition:
-                if self.lexico_objectives is None:
+                if not self.lexico_objectives:
                     obj = result[self._search_alg.metric] * self._metric_op
                 else:
                     obj = {}
                     for k, m in zip(
                         self._search_alg.lexico_objectives["metrics"], self._search_alg.lexico_objectives["modes"]
                     ):
-                        obj[k] = -result[k] if m == "max" else result[k]
+                        obj[k] = -1 * result[k] if m == "max" else result[k]
                 if self.best_result is None or self._better(obj, self.obj_best1):
                     self.cost_best2 = self.cost_best1
                     self.cost_best1 = self.cost_total
-                    if self.lexico_objectives is None:
+                    if not self.lexico_objectives:
                         self.obj_best2 = obj if np.isinf(self.obj_best1) else self.obj_best1
                     else:
                         self.obj_best2 = (
diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py
index 52a0c47eb7..50adb04969 100644
--- a/flaml/tune/tune.py
+++ b/flaml/tune/tune.py
@@ -52,14 +52,14 @@ def __init__(self, trials, metric, mode, lexico_objectives=None):
 
     @property
     def best_trial(self) -> Trial:
-        if self.lexico_objectives is None:
+        if not self.lexico_objectives:
             return super().best_trial
         else:
             return self.get_best_trial(self.default_metric, self.default_mode)
 
     @property
     def best_config(self) -> Dict:
-        if self.lexico_objectives is None:
+        if not self.lexico_objectives:
             return super().best_config
         else:
             return self.get_best_config(self.default_metric, self.default_mode)
@@ -110,7 +110,7 @@ def get_best_trial(
         scope: str = "last",
         filter_nan_and_inf: bool = True,
     ) -> Optional[Trial]:
-        if self.lexico_objectives is not None:
+        if self.lexico_objectives:
             best_trial = self.lexico_best(self.trials)
         else:
             best_trial = super().get_best_trial(metric, mode, scope, filter_nan_and_inf)

From 588de22ca56972b1b97cf1f2ff7cb26b3ef79042 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Fri, 16 Jun 2023 16:36:42 +0800
Subject: [PATCH 08/15] update

---
 flaml/tune/searcher/blendsearch.py | 3 ++-
 flaml/tune/searcher/suggestion.py  | 1 -
 test/tune/test_lexiflow.py         | 2 ++
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index eec85097ac..bee19ed1c2 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -677,7 +677,7 @@ def _create_condition(self, result: Dict) -> bool:
             return result[self._ls.metric] * self._ls.metric_op < obj_median
         else:
             thread_pools = [thread.obj_best1 for id, thread in self._search_thread_pool.items() if id]
-            self._lexico_sort(thread_pools)
+            thread_pools = self._lexico_sort(thread_pools)
             obj_median = thread_pools[round(len(thread_pools) / 2)]
             result = self._unify_op(result)
             return self._lexico_inferior(obj_median, result)
@@ -797,6 +797,7 @@ def _lexico_sort(self, arr: list):
             for j in range(0, n - i - 1):
                 if self._lexico_(arr[j], arr[j + 1]):
                     arr[j], arr[j + 1] = arr[j + 1], arr[j]
+        return arr
 
     def _inferior(self, id1: int, id2: int) -> bool:
         """whether thread id1 is inferior to id2"""
diff --git a/flaml/tune/searcher/suggestion.py b/flaml/tune/searcher/suggestion.py
index fc1b92c153..4e8f807c29 100644
--- a/flaml/tune/searcher/suggestion.py
+++ b/flaml/tune/searcher/suggestion.py
@@ -34,7 +34,6 @@
     Uniform,
 )
 from ..trial import flatten_dict, unflatten_dict
-
 from ray import __version__ as ray_version
 
 assert ray_version >= "1.10.0"
diff --git a/test/tune/test_lexiflow.py b/test/tune/test_lexiflow.py
index 2d0274634a..f5bb12bf66 100644
--- a/test/tune/test_lexiflow.py
+++ b/test/tune/test_lexiflow.py
@@ -103,6 +103,7 @@ def evaluate_function(configuration):
 
     lexico_objectives = {}
     lexico_objectives["metrics"] = ["error_rate", "flops"]
+    lexico_objectives["algorithm"] = ["CFO"]
 
     search_space = {
         "n_layers": tune.randint(lower=1, upper=3),
@@ -178,6 +179,7 @@ def test_lexiflow_performance():
     lexico_objectives["tolerances"] = {"brain": 10.0, "currin": 0.0}
     lexico_objectives["targets"] = {"brain": 0.0, "currin": 0.0}
     lexico_objectives["modes"] = ["min", "min"]
+    lexico_objectives["algorithm"] = ["CFO"]
 
     search_space = {
         "x1": tune.uniform(lower=0.000001, upper=1.0),

From 6158bacd00040a3c3960beac082cb66938e441d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Fri, 16 Jun 2023 17:00:31 +0800
Subject: [PATCH 09/15] update

---
 flaml/tune/searcher/suggestion.py | 425 +++++++++++++++++++++---------
 1 file changed, 300 insertions(+), 125 deletions(-)

diff --git a/flaml/tune/searcher/suggestion.py b/flaml/tune/searcher/suggestion.py
index 4e8f807c29..565640c390 100644
--- a/flaml/tune/searcher/suggestion.py
+++ b/flaml/tune/searcher/suggestion.py
@@ -19,8 +19,8 @@
 import functools
 import warnings
 import copy
-import logging
 import numpy as np
+import logging
 from typing import Any, Dict, Optional, Union, List, Tuple, Callable
 import pickle
 from .variant_generator import parse_spec_vars
@@ -34,13 +34,7 @@
     Uniform,
 )
 from ..trial import flatten_dict, unflatten_dict
-from ray import __version__ as ray_version
 
-assert ray_version >= "1.10.0"
-if ray_version.startswith("1."):
-    from ray.tune.suggest.optuna import OptunaSearch as MOSearch
-else:
-    from ray.tune.search.optuna import OptunaSearch as MOSearch
 logger = logging.getLogger(__name__)
 
 UNRESOLVED_SEARCH_SPACE = str(
@@ -345,81 +339,234 @@ def __getattr__(self, item_name: str) -> Any:
 
 class OptunaSearch(Searcher):
     """A wrapper around Optuna to provide trial suggestions.
-        [Optuna](https://optuna.org/)
-        is a hyperparameter optimization library.
-        In contrast to other libraries, it employs define-by-run style
-        hyperparameter definitions.
-        This Searcher is a thin wrapper around Optuna's search algorithms.
-        You can pass any Optuna sampler, which will be used to generate
-        hyperparameter suggestions.
-        Args:
-            space (dict|Callable): Hyperparameter search space definition for
-                Optuna's sampler. This can be either a class `dict` with
-                parameter names as keys and ``optuna.distributions`` as values,
-                or a Callable - in which case, it should be a define-by-run
-                function using ``optuna.trial`` to obtain the hyperparameter
-                values. The function should return either a class `dict` of
-                constant values with names as keys, or None.
-                For more information, see
-                [tutorial](https://optuna.readthedocs.io/en/stable/tutorial/10_key_features/002_configurations.html).
-                Warning - No actual computation should take place in the define-by-run
+
+    `Optuna <https://optuna.org/>`_ is a hyperparameter optimization library.
+    In contrast to other libraries, it employs define-by-run style
+    hyperparameter definitions.
+
+    This Searcher is a thin wrapper around Optuna's search algorithms.
+    You can pass any Optuna sampler, which will be used to generate
+    hyperparameter suggestions.
+
+    Multi-objective optimization is supported.
+
+    Args:
+        space: Hyperparameter search space definition for
+            Optuna's sampler. This can be either a :class:`dict` with
+            parameter names as keys and ``optuna.distributions`` as values,
+            or a Callable - in which case, it should be a define-by-run
+            function using ``optuna.trial`` to obtain the hyperparameter
+            values. The function should return either a :class:`dict` of
+            constant values with names as keys, or None.
+            For more information, see https://optuna.readthedocs.io\
+/en/stable/tutorial/10_key_features/002_configurations.html.
+
+            .. warning::
+                No actual computation should take place in the define-by-run
                 function. Instead, put the training logic inside the function
-                or class trainable passed to tune.run.
-            metric (str): The training result objective value attribute. If None
-                but a mode was passed, the anonymous metric `_metric` will be used
-                per default.
-            mode (str): One of {min, max}. Determines whether objective is
-                minimizing or maximizing the metric attribute.
-            points_to_evaluate (list): Initial parameter suggestions to be run
-                first. This is for when you already have some good parameters
-                you want to run first to help the algorithm make better suggestions
-                for future parameters. Needs to be a list of dicts containing the
-                configurations.
-            sampler (optuna.samplers.BaseSampler): Optuna sampler used to
-                draw hyperparameter configurations. Defaults to ``TPESampler``.
-            seed (int): Seed to initialize sampler with. This parameter is only
-                used when ``sampler=None``. In all other cases, the sampler
-                you pass should be initialized with the seed already.
-            evaluated_rewards (list): If you have previously evaluated the
-                parameters passed in as points_to_evaluate you can avoid
-                re-running those trials by passing in the reward attributes
-                as a list so the optimiser can be told the results without
-                needing to re-compute the trial. Must be the same length as
-                points_to_evaluate.
-
-        Tune automatically converts search spaces to Optuna's format:
-
-    ````python
-    from ray.tune.suggest.optuna import OptunaSearch  # ray version < 2
-    config = { "a": tune.uniform(6, 8),
-               "b": tune.loguniform(1e-4, 1e-2)}
-    optuna_search = OptunaSearch(metric="loss", mode="min")
-    tune.run(trainable, config=config, search_alg=optuna_search)
-    ````
-
-        If you would like to pass the search space manually, the code would
-        look like this:
+                or class trainable passed to ``tune.Tuner()``.
+
+        metric: The training result objective value attribute. If
+            None but a mode was passed, the anonymous metric ``_metric``
+            will be used per default. Can be a list of metrics for
+            multi-objective optimization.
+        mode: One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute. Can be a list of
+            modes for multi-objective optimization (corresponding to
+            ``metric``).
+        points_to_evaluate: Initial parameter suggestions to be run
+            first. This is for when you already have some good parameters
+            you want to run first to help the algorithm make better suggestions
+            for future parameters. Needs to be a list of dicts containing the
+            configurations.
+        sampler: Optuna sampler used to
+            draw hyperparameter configurations. Defaults to ``MOTPESampler``
+            for multi-objective optimization with Optuna<2.9.0, and
+            ``TPESampler`` in every other case.
+            See https://optuna.readthedocs.io/en/stable/reference/samplers/index.html
+            for available Optuna samplers.
+
+            .. warning::
+                Please note that with Optuna 2.10.0 and earlier
+                default ``MOTPESampler``/``TPESampler`` suffer
+                from performance issues when dealing with a large number of
+                completed trials (approx. >100). This will manifest as
+                a delay when suggesting new configurations.
+                This is an Optuna issue and may be fixed in a future
+                Optuna release.
+
+        seed: Seed to initialize sampler with. This parameter is only
+            used when ``sampler=None``. In all other cases, the sampler
+            you pass should be initialized with the seed already.
+        evaluated_rewards: If you have previously evaluated the
+            parameters passed in as points_to_evaluate you can avoid
+            re-running those trials by passing in the reward attributes
+            as a list so the optimiser can be told the results without
+            needing to re-compute the trial. Must be the same length as
+            points_to_evaluate.
+
+            .. warning::
+                When using ``evaluated_rewards``, the search space ``space``
+                must be provided as a :class:`dict` with parameter names as
+                keys and ``optuna.distributions`` instances as values. The
+                define-by-run search space definition is not yet supported with
+                this functionality.
+
+    Tune automatically converts search spaces to Optuna's format:
+
+    .. code-block:: python
+
+        from ray.tune.search.optuna import OptunaSearch
+
+        config = {
+            "a": tune.uniform(6, 8)
+            "b": tune.loguniform(1e-4, 1e-2)
+        }
+
+        optuna_search = OptunaSearch(
+            metric="loss",
+            mode="min")
+
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=optuna_search,
+            ),
+            param_space=config,
+        )
+        tuner.fit()
+
+    If you would like to pass the search space manually, the code would
+    look like this:
+
+    .. code-block:: python
+
+        from ray.tune.search.optuna import OptunaSearch
+        import optuna
+
+        space = {
+            "a": optuna.distributions.UniformDistribution(6, 8),
+            "b": optuna.distributions.LogUniformDistribution(1e-4, 1e-2),
+        }
+
+        optuna_search = OptunaSearch(
+            space,
+            metric="loss",
+            mode="min")
+
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=optuna_search,
+            ),
+        )
+        tuner.fit()
+
+        # Equivalent Optuna define-by-run function approach:
+
+        def define_search_space(trial: optuna.Trial):
+            trial.suggest_float("a", 6, 8)
+            trial.suggest_float("b", 1e-4, 1e-2, log=True)
+            # training logic goes into trainable, this is just
+            # for search space definition
+
+        optuna_search = OptunaSearch(
+            define_search_space,
+            metric="loss",
+            mode="min")
+
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=optuna_search,
+            ),
+        )
+        tuner.fit()
+
+    Multi-objective optimization is supported:
+
+    .. code-block:: python
+
+        from ray.tune.search.optuna import OptunaSearch
+        import optuna
+
+        space = {
+            "a": optuna.distributions.UniformDistribution(6, 8),
+            "b": optuna.distributions.LogUniformDistribution(1e-4, 1e-2),
+        }
+
+        # Note you have to specify metric and mode here instead of
+        # in tune.TuneConfig
+        optuna_search = OptunaSearch(
+            space,
+            metric=["loss1", "loss2"],
+            mode=["min", "max"])
+
+        # Do not specify metric and mode here!
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=optuna_search,
+            ),
+        )
+        tuner.fit()
+
+    You can pass configs that will be evaluated first using
+    ``points_to_evaluate``:
+
+    .. code-block:: python
+
+        from ray.tune.search.optuna import OptunaSearch
+        import optuna
+
+        space = {
+            "a": optuna.distributions.UniformDistribution(6, 8),
+            "b": optuna.distributions.LogUniformDistribution(1e-4, 1e-2),
+        }
+
+        optuna_search = OptunaSearch(
+            space,
+            points_to_evaluate=[{"a": 6.5, "b": 5e-4}, {"a": 7.5, "b": 1e-3}]
+            metric="loss",
+            mode="min")
+
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=optuna_search,
+            ),
+        )
+        tuner.fit()
+
+    Avoid re-running evaluated trials by passing the rewards together with
+    `points_to_evaluate`:
+
+    .. code-block:: python
+
+        from ray.tune.search.optuna import OptunaSearch
+        import optuna
+
+        space = {
+            "a": optuna.distributions.UniformDistribution(6, 8),
+            "b": optuna.distributions.LogUniformDistribution(1e-4, 1e-2),
+        }
+
+        optuna_search = OptunaSearch(
+            space,
+            points_to_evaluate=[{"a": 6.5, "b": 5e-4}, {"a": 7.5, "b": 1e-3}]
+            evaluated_rewards=[0.89, 0.42]
+            metric="loss",
+            mode="min")
+
+        tuner = tune.Tuner(
+            trainable,
+            tune_config=tune.TuneConfig(
+                search_alg=optuna_search,
+            ),
+        )
+        tuner.fit()
 
-    ```python
-    from ray.tune.suggest.optuna import OptunaSearch  # ray version < 2
-    import optuna
-    config = { "a": optuna.distributions.UniformDistribution(6, 8),
-               "b": optuna.distributions.LogUniformDistribution(1e-4, 1e-2)}
-    optuna_search = OptunaSearch(space,metric="loss",mode="min")
-    tune.run(trainable, search_alg=optuna_search)
-    # Equivalent Optuna define-by-run function approach:
-    def define_search_space(trial: optuna.Trial):
-        trial.suggest_float("a", 6, 8)
-        trial.suggest_float("b", 1e-4, 1e-2, log=True)
-        # training logic goes into trainable, this is just
-        # for search space definition
-    optuna_search = OptunaSearch(
-        define_search_space,
-        metric="loss",
-        mode="min")
-    tune.run(trainable, search_alg=optuna_search)
     .. versionadded:: 0.8.8
-    ```
 
     """
 
@@ -432,15 +579,15 @@ def __init__(
                 Callable[["OptunaTrial"], Optional[Dict[str, Any]]],
             ]
         ] = None,
-        metric: Optional[str] = None,
-        mode: Optional[str] = None,
+        metric: Optional[Union[str, List[str]]] = None,
+        mode: Optional[Union[str, List[str]]] = None,
         points_to_evaluate: Optional[List[Dict]] = None,
         sampler: Optional["BaseSampler"] = None,
         seed: Optional[int] = None,
         evaluated_rewards: Optional[List] = None,
     ):
         assert ot is not None, "Optuna must be installed! Run `pip install optuna`."
-        super(OptunaSearch, self).__init__(metric=metric, mode=mode, max_concurrent=None, use_early_stopped_trials=None)
+        super(OptunaSearch, self).__init__(metric=metric, mode=mode)
 
         if isinstance(space, dict) and space:
             resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
@@ -464,33 +611,55 @@ def __init__(
                 "`seed` parameter has to be passed to the sampler directly "
                 "and will be ignored."
             )
+        elif sampler:
+            assert isinstance(sampler, BaseSampler), (
+                "You can only pass an instance of " "`optuna.samplers.BaseSampler` " "as a sampler to `OptunaSearcher`."
+            )
 
-        self._sampler = sampler or ot.samplers.TPESampler(seed=seed)
+        self._sampler = sampler
+        self._seed = seed
 
-        assert isinstance(self._sampler, BaseSampler), (
-            "You can only pass an instance of `optuna.samplers.BaseSampler` " "as a sampler to `OptunaSearcher`."
-        )
+        self._completed_trials = set()
 
         self._ot_trials = {}
         self._ot_study = None
         if self._space:
             self._setup_study(mode)
 
-    def _setup_study(self, mode: str):
+    def _setup_study(self, mode: Union[str, list]):
         if self._metric is None and self._mode:
+            if isinstance(self._mode, list):
+                raise ValueError(
+                    "If ``mode`` is a list (multi-objective optimization " "case), ``metric`` must be defined."
+                )
             # If only a mode was passed, use anonymous metric
             self._metric = DEFAULT_METRIC
 
         pruner = ot.pruners.NopPruner()
         storage = ot.storages.InMemoryStorage()
 
+        if self._sampler:
+            sampler = self._sampler
+        elif isinstance(mode, list):
+            # MOTPESampler deprecated in Optuna>=2.9.0
+            sampler = ot.samplers.MOTPESampler(seed=self._seed)
+
+        if isinstance(mode, list):
+            study_direction_args = dict(
+                directions=["minimize" if m == "min" else "maximize" for m in mode],
+            )
+        else:
+            study_direction_args = dict(
+                direction="minimize" if mode == "min" else "maximize",
+            )
+
         self._ot_study = ot.study.create_study(
             storage=storage,
-            sampler=self._sampler,
+            sampler=sampler,
             pruner=pruner,
             study_name=self._study_name,
-            direction="minimize" if mode == "min" else "maximize",
             load_if_exists=True,
+            **study_direction_args,
         )
 
         if self._points_to_evaluate:
@@ -507,7 +676,7 @@ def _setup_study(self, mode: str):
                 for point in self._points_to_evaluate:
                     self._ot_study.enqueue_trial(point)
 
-    def set_search_properties(self, metric: Optional[str], mode: Optional[str], config: Dict) -> bool:
+    def set_search_properties(self, metric: Optional[str], mode: Optional[str], config: Dict, **spec) -> bool:
         if self._space:
             return False
         space = self.convert_search_space(config)
@@ -517,7 +686,7 @@ def set_search_properties(self, metric: Optional[str], mode: Optional[str], conf
         if mode:
             self._mode = mode
 
-        self._setup_study(mode)
+        self._setup_study(self._mode)
         return True
 
     def _suggest_from_define_by_run_func(
@@ -535,7 +704,7 @@ def _suggest_from_define_by_run_func(
                 f"took {time_taken} seconds to "
                 "run. Ensure that actual computation, training takes "
                 "place inside Tune's train functions or Trainables "
-                "passed to `tune.run`."
+                "passed to `tune.Tuner()`."
             )
         if ret is not None:
             if not isinstance(ret, dict):
@@ -560,21 +729,8 @@ def suggest(self, trial_id: str) -> Optional[Dict]:
             raise RuntimeError(
                 UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)
             )
-
-        if isinstance(self._space, list):
-            # Keep for backwards compatibility
-            # Deprecate: 1.5
-            if trial_id not in self._ot_trials:
-                self._ot_trials[trial_id] = self._ot_study.ask()
-
-            ot_trial = self._ot_trials[trial_id]
-
-            # getattr will fetch the trial.suggest_ function on Optuna trials
-            params = {
-                args[0] if len(args) > 0 else kwargs["name"]: getattr(ot_trial, fn)(*args, **kwargs)
-                for (fn, args, kwargs) in self._space
-            }
-        elif callable(self._space):
+        if callable(self._space):
+            # Define-by-run case
             if trial_id not in self._ot_trials:
                 self._ot_trials[trial_id] = self._ot_study.ask()
 
@@ -591,15 +747,36 @@ def suggest(self, trial_id: str) -> Optional[Dict]:
         return unflatten_dict(params)
 
     def on_trial_result(self, trial_id: str, result: Dict):
+        if isinstance(self.metric, list):
+            # Optuna doesn't support incremental results
+            # for multi-objective optimization
+            return
+        if trial_id in self._completed_trials:
+            logger.warning(
+                f"Received additional result for trial {trial_id}, but " f"it already finished. Result: {result}"
+            )
+            return
         metric = result[self.metric]
         step = result[TRAINING_ITERATION]
         ot_trial = self._ot_trials[trial_id]
         ot_trial.report(metric, step)
 
     def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
+        if trial_id in self._completed_trials:
+            logger.warning(
+                f"Received additional completion for trial {trial_id}, but " f"it already finished. Result: {result}"
+            )
+            return
+
         ot_trial = self._ot_trials[trial_id]
 
-        val = result.get(self.metric, None) if result else None
+        if result:
+            if isinstance(self.metric, list):
+                val = [result.get(metric, None) for metric in self.metric]
+            else:
+                val = result.get(self.metric, None)
+        else:
+            val = None
         ot_trial_state = OptunaTrialState.COMPLETE
         if val is None:
             if error:
@@ -608,9 +785,11 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
                 ot_trial_state = OptunaTrialState.PRUNED
         try:
             self._ot_study.tell(ot_trial, val, state=ot_trial_state)
-        except ValueError as exc:
+        except Exception as exc:
             logger.warning(exc)  # E.g. if NaN was reported
 
+        self._completed_trials.add(trial_id)
+
     def add_evaluated_point(
         self,
         parameters: Dict,
@@ -625,6 +804,13 @@ def add_evaluated_point(
             raise RuntimeError(
                 UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)
             )
+        if callable(self._space):
+            raise TypeError(
+                "Define-by-run function passed in `space` argument is not "
+                "yet supported when using `evaluated_rewards`. Please provide "
+                "an `OptunaDistribution` dict or pass a Ray Tune "
+                "search space to `tune.Tuner()`."
+            )
 
         ot_trial_state = OptunaTrialState.COMPLETE
         if error:
@@ -648,27 +834,15 @@ def add_evaluated_point(
         self._ot_study.add_trial(trial)
 
     def save(self, checkpoint_path: str):
-        save_object = (
-            self._sampler,
-            self._ot_trials,
-            self._ot_study,
-            self._points_to_evaluate,
-            self._evaluated_rewards,
-        )
+        save_object = self.__dict__.copy()
         with open(checkpoint_path, "wb") as outputFile:
             pickle.dump(save_object, outputFile)
 
     def restore(self, checkpoint_path: str):
         with open(checkpoint_path, "rb") as inputFile:
             save_object = pickle.load(inputFile)
-        if len(save_object) == 5:
-            (
-                self._sampler,
-                self._ot_trials,
-                self._ot_study,
-                self._points_to_evaluate,
-                self._evaluated_rewards,
-            ) = save_object
+        if isinstance(save_object, dict):
+            self.__dict__.update(save_object)
         else:
             # Backwards compatibility
             (
@@ -676,6 +850,7 @@ def restore(self, checkpoint_path: str):
                 self._ot_trials,
                 self._ot_study,
                 self._points_to_evaluate,
+                self._evaluated_rewards,
             ) = save_object
 
     @staticmethod
@@ -748,7 +923,7 @@ def resolve_value(domain: Domain) -> ot.distributions.BaseDistribution:
         return values
 
 
-class LexiGlobalSearch(MOSearch):
+class LexiGlobalSearch(OptunaSearch):
     def __init__(
         self,
         space: Optional[

From 36d616db8fdc773403be8fa020ff4104e169c7d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Wed, 21 Jun 2023 22:06:56 +0800
Subject: [PATCH 10/15] 1. remove redundant get_lexibound 2. use origin optuna
 searcher

---
 flaml/tune/searcher/blendsearch.py   | 22 ++----------
 flaml/tune/searcher/flow2.py         | 52 +++++++++++++---------------
 flaml/tune/searcher/search_thread.py |  4 ++-
 flaml/tune/searcher/suggestion.py    | 23 +++---------
 flaml/tune/utils.py                  | 13 +++++++
 5 files changed, 48 insertions(+), 66 deletions(-)

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index bee19ed1c2..771dfe3b1c 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -8,6 +8,7 @@
 import time
 import pickle
 
+
 try:
     from ray import __version__ as ray_version
 
@@ -28,6 +29,7 @@
 from .flow2 import FLOW2
 from ..space import add_cost_to_space, indexof, normalize, define_by_run_func
 from ..result import TIME_TOTAL_S
+from ..utils import get_lexico_bound
 
 import logging
 
@@ -487,24 +489,6 @@ def restore(self, checkpoint_path: str):
         self._start_time = time.time()
         self._set_deadline()
 
-    def _get_lexico_bound(self, metric, mode):
-        k_target = (
-            self.lexico_objectives["targets"][metric]
-            if mode == "min"
-            else -1 * self.lexico_objectives["targets"][metric]
-        )
-        if not isinstance(self.lexico_objectives["tolerances"][metric], str):
-            tolerance_bound = self._f_best[metric] + self.lexico_objectives["tolerances"][metric]
-        else:
-            assert (
-                self.lexico_objectives["tolerances"][metric][-1] == "%"
-            ), "String tolerance of {} should use %% as the suffix".format(metric)
-            tolerance_bound = self._f_best[metric] * (
-                1 + 0.01 * float(self.lexico_objectives["tolerances"][metric].replace("%", ""))
-            )
-        bound = max(tolerance_bound, k_target)
-        return bound
-
     @property
     def metric_target(self):
         return self._metric_target
@@ -763,7 +747,7 @@ def _expand_admissible_region(self, lower, upper, space):
     def _lexico_inferior(self, obj_1: Union[dict, float], obj_2: Union[dict, float]) -> bool:
         if self.lexico_objectives:
             for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                bound = self._get_lexico_bound(k_metric, k_mode)
+                bound = get_lexico_bound(k_metric, k_mode, self.lexico_objectives, self._f_best)
                 if (obj_1[k_metric] < bound) and (obj_2[k_metric] < bound):
                     continue
                 elif obj_1[k_metric] < obj_2[k_metric]:
diff --git a/flaml/tune/searcher/flow2.py b/flaml/tune/searcher/flow2.py
index 4e8e4375bc..1aa44b5c8b 100644
--- a/flaml/tune/searcher/flow2.py
+++ b/flaml/tune/searcher/flow2.py
@@ -53,6 +53,7 @@ def __init__(
         lexico_objectives=None,
     ):
         """Constructor.
+
         Args:
             init_config: a dictionary of a partial or full initial config,
                 e.g., from a subset of controlled dimensions
@@ -184,7 +185,6 @@ def _init_search(self):
         self.incumbent = {}
         self.incumbent = self.normalize(self.best_config)  # flattened
         self.best_obj = self.cost_incumbent = None
-        # self.pre_best_obj = None
         self.dim = len(self._tunable_keys)  # total # tunable dimensions
         self._direction_tried = None
         self._num_complete4incumbent = self._cost_complete4incumbent = 0
@@ -266,6 +266,7 @@ def complete_config(
         upper: Optional[Dict] = None,
     ) -> Tuple[Dict, Dict]:
         """Generate a complete config from the partial config input.
+
         Add minimal resource to config if available.
         """
         disturb = self._reset_times and partial_config == self.init_config
@@ -292,13 +293,11 @@ def create(self, init_config: Dict, obj: float, cost: float, space: Dict) -> Sea
             self.seed + 1,
             self.lexico_objectives,
         )
-        if self.lexico_objectives:
+        if self.lexico_objectives is not None:
             flow2.best_obj = {}
             for k, v in obj.items():
                 flow2.best_obj[k] = (
-                    -1 * v
-                    if self.lexico_objectives["modes"][self.lexico_objectives["metrics"].index(k)] == "max"
-                    else v
+                    -v if self.lexico_objectives["modes"][self.lexico_objectives["metrics"].index(k)] == "max" else v
                 )
         else:
             flow2.best_obj = obj * self.metric_op  # minimize internally
@@ -314,24 +313,6 @@ def denormalize(self, config):
         """denormalize each dimension in config from [0,1]."""
         return denormalize(config, self._space, self.best_config, self.incumbent, self._random)
 
-    def _get_lexico_bound(self, metric, mode):
-        k_target = (
-            self.lexico_objectives["targets"][metric]
-            if mode == "min"
-            else -1 * self.lexico_objectives["targets"][metric]
-        )
-        if not isinstance(self.lexico_objectives["tolerances"][metric], str):
-            tolerance_bound = self._f_best[metric] + self.lexico_objectives["tolerances"][metric]
-        else:
-            assert (
-                self.lexico_objectives["tolerances"][metric][-1] == "%"
-            ), "String tolerance of {} should use %% as the suffix".format(metric)
-            tolerance_bound = self._f_best[metric] * (
-                1 + 0.01 * float(self.lexico_objectives["tolerances"][metric].replace("%", ""))
-            )
-        bound = max(tolerance_bound, k_target)
-        return bound
-
     def set_search_properties(
         self,
         metric: Optional[str] = None,
@@ -392,8 +373,27 @@ def lexico_compare(self, result) -> bool:
                 self._histories[k].append(result[k])
             self.update_fbest()
             for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                bound = self._get_lexico_bound(k_metric, k_mode)
-                if (result[k_metric] < bound) and (self.best_obj[k_metric] < bound):
+                k_target = (
+                    self.lexico_objectives["targets"][k_metric]
+                    if k_mode == "min"
+                    else -self.lexico_objectives["targets"][k_metric]
+                )
+                if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
+                    tolerance_bound = self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
+                else:
+                    assert (
+                        self.lexico_objectives["tolerances"][k_metric][-1] == "%"
+                    ), "String tolerance of {} should use %% as the suffix".format(k_metric)
+                    tolerance_bound = self._f_best[k_metric] * (
+                        1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))
+                    )
+                if (result[k_metric] < max(tolerance_bound, k_target)) and (
+                    self.best_obj[k_metric]
+                    < max(
+                        tolerance_bound,
+                        k_target,
+                    )
+                ):
                     continue
                 elif result[k_metric] < self.best_obj[k_metric]:
                     self.op_dimension = k_metric
@@ -439,7 +439,6 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
                     or (self.lexico_objectives is None and obj < self.best_obj)
                     or (self.lexico_objectives is not None and self.lexico_compare(obj))
                 ):
-                    # self.pre_best_obj = self.best_obj
                     self.best_obj = obj
                     self.best_config, self.step = self._configs[trial_id]
                     self.incumbent = self.normalize(self.best_config)
@@ -497,7 +496,6 @@ def on_trial_result(self, trial_id: str, result: Dict):
                     or (self.lexico_objectives is None and obj < self.best_obj)
                     or (self.lexico_objectives is not None and self.lexico_compare(obj))
                 ):
-                    # self.pre_best_obj = self.best_obj
                     self.best_obj = obj
                     config = self._configs[trial_id][0]
                     if self.best_config != config:
diff --git a/flaml/tune/searcher/search_thread.py b/flaml/tune/searcher/search_thread.py
index 5aa137360a..a3cfabd110 100644
--- a/flaml/tune/searcher/search_thread.py
+++ b/flaml/tune/searcher/search_thread.py
@@ -5,6 +5,7 @@
 from typing import Dict, Optional, Union
 import numpy as np
 
+
 try:
     from ray import __version__ as ray_version
 
@@ -18,6 +19,7 @@
 from .flow2 import FLOW2
 from ..space import add_cost_to_space, unflatten_hierarchical
 from ..result import TIME_TOTAL_S
+from ..utils import get_lexico_bound
 import logging
 
 logger = logging.getLogger(__name__)
@@ -121,7 +123,7 @@ def update_eci(self, metric_target: float, max_speed: Optional[float] = np.inf,
     def _better(self, obj_1: Union[dict, float], obj_2: Union[dict, float]):
         if self.lexico_objectives:
             for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                bound = self._search_alg._get_lexico_bound(k_metric, k_mode)
+                bound = get_lexico_bound(k_metric, k_mode, self.lexico_objectives, self._search_alg._f_best)
                 if (obj_1[k_metric] < bound) and (obj_2[k_metric] < bound):
                     continue
                 elif obj_1[k_metric] < obj_2[k_metric]:
diff --git a/flaml/tune/searcher/suggestion.py b/flaml/tune/searcher/suggestion.py
index 565640c390..2efccb94e8 100644
--- a/flaml/tune/searcher/suggestion.py
+++ b/flaml/tune/searcher/suggestion.py
@@ -34,6 +34,7 @@
     Uniform,
 )
 from ..trial import flatten_dict, unflatten_dict
+from packaging import version
 
 logger = logging.getLogger(__name__)
 
@@ -640,9 +641,11 @@ def _setup_study(self, mode: Union[str, list]):
 
         if self._sampler:
             sampler = self._sampler
-        elif isinstance(mode, list):
+        elif isinstance(mode, list) and version.parse(ot.__version__) < version.parse("2.9.0"):
             # MOTPESampler deprecated in Optuna>=2.9.0
             sampler = ot.samplers.MOTPESampler(seed=self._seed)
+        else:
+            sampler = ot.samplers.TPESampler(seed=self._seed)
 
         if isinstance(mode, list):
             study_direction_args = dict(
@@ -985,24 +988,6 @@ def update_fbest(
             )[0]
             feasible_index = feasible_index.take(feasible_index_filter)
 
-    def _get_lexico_bound(self, metric, mode):
-        k_target = (
-            self.lexico_objectives["targets"][metric]
-            if mode == "min"
-            else -1 * self.lexico_objectives["targets"][metric]
-        )
-        if not isinstance(self.lexico_objectives["tolerances"][metric], str):
-            tolerance_bound = self._f_best[metric] + self.lexico_objectives["tolerances"][metric]
-        else:
-            assert (
-                self.lexico_objectives["tolerances"][metric][-1] == "%"
-            ), "String tolerance of {} should use %% as the suffix".format(metric)
-            tolerance_bound = self._f_best[metric] * (
-                1 + 0.01 * float(self.lexico_objectives["tolerances"][metric].replace("%", ""))
-            )
-        bound = max(tolerance_bound, k_target)
-        return bound
-
     def on_trial_result(self, trial_id: str, result: Dict):
         super.on_trial_result(trial_id, result)
         for k_metric, k_mode in zip(self.lexico_objectives["metrics"]):
diff --git a/flaml/tune/utils.py b/flaml/tune/utils.py
index 9398162a3c..96b86ec27d 100644
--- a/flaml/tune/utils.py
+++ b/flaml/tune/utils.py
@@ -25,3 +25,16 @@ def choice(categories: Sequence, order=None):
     domain = sample.Categorical(categories).uniform()
     domain.ordered = order if order is not None else all(isinstance(x, (int, float)) for x in categories)
     return domain
+
+
+def get_lexico_bound(metric, mode, lexico_objectives, f_best):
+    k_target = lexico_objectives["targets"][metric] if mode == "min" else -1 * lexico_objectives["targets"][metric]
+    if not isinstance(lexico_objectives["tolerances"][metric], str):
+        tolerance_bound = f_best[metric] + lexico_objectives["tolerances"][metric]
+    else:
+        assert (
+            lexico_objectives["tolerances"][metric][-1] == "%"
+        ), "String tolerance of {} should use %% as the suffix".format(metric)
+        tolerance_bound = f_best[metric] * (1 + 0.01 * float(lexico_objectives["tolerances"][metric].replace("%", "")))
+    bound = max(tolerance_bound, k_target)
+    return bound

From f070d80a9f6ee8adc15c77c814abbc1883e3a6ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Sat, 24 Jun 2023 23:40:16 +0800
Subject: [PATCH 11/15] set init blendsearch

---
 flaml/tune/searcher/blendsearch.py | 158 +++++++++++------------------
 flaml/tune/utils.py                |   3 +
 2 files changed, 65 insertions(+), 96 deletions(-)

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index 771dfe3b1c..93bc743416 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -15,13 +15,13 @@
     assert ray_version >= "1.10.0"
     if ray_version.startswith("1."):
         from ray.tune.suggest import Searcher
-        from ray.tune.suggest.optuna import OptunaSearch as GlobalSearch
+        from ray.tune.suggest.optuna import OptunaSearch as NormalGlobalSearch
     else:
         from ray.tune.search import Searcher
-        from ray.tune.search.optuna import OptunaSearch as GlobalSearch
+        from ray.tune.search.optuna import OptunaSearch as NormalGlobalSearch
 except (ImportError, AssertionError):
     from .suggestion import Searcher
-    from .suggestion import OptunaSearch as GlobalSearch
+    from .suggestion import OptunaSearch as NormalGlobalSearch
 from .suggestion import LexiGlobalSearch as LexiGlobalSearch
 from ..trial import unflatten_dict, flatten_dict
 from .. import INCUMBENT_RESULT
@@ -36,6 +36,7 @@
 SEARCH_THREAD_EPS = 1.0
 PENALTY = 1e10  # penalty term for constraints
 logger = logging.getLogger(__name__)
+GlobalSearch = NormalGlobalSearch
 
 
 class BlendSearch(Searcher):
@@ -165,14 +166,7 @@ def __init__(
             self.cost_attr = cost_attr
             self._cost_budget = cost_budget
         self.penalty = PENALTY  # penalty term for constraints
-        if not self.lexico_objectives:
-            self._metric, self._mode = metric, mode
-        else:
-            self._metric = self.lexico_objectives["metrics"][0]
-            self._mode = self.lexico_objectives["modes"][0]
         self._use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
-        self._histories = None
-        self._f_best = None
         init_config = low_cost_partial_config or {}
         if not init_config:
             logger.info(
@@ -203,16 +197,10 @@ def __init__(
             self._evaluated_rewards = evaluated_rewards or []
         self._config_constraints = config_constraints
         self._metric_constraints = metric_constraints
-        if metric_constraints:
-            if self.lexico_objectives:
-                self._metric_constraints = None
-                logger.info("Do not support providing metric_constraints in lexicographic optimization for now.")
-            else:
-                assert all(
-                    x[1] in ["<=", ">="] for x in metric_constraints
-                ), "sign of metric constraints must be <= or >=."
-                # metric modified by lagrange
-                metric += self.lagrange
+        if metric_constraints and not self.lexico_objectives:
+            assert all(x[1] in ["<=", ">="] for x in metric_constraints), "sign of metric constraints must be <= or >=."
+            # metric modified by lagrange
+            metric += self.lagrange
         self._cat_hp_cost = cat_hp_cost or {}
         if space:
             add_cost_to_space(space, init_config, self._cat_hp_cost)
@@ -241,6 +229,9 @@ def __init__(
                 gs_space = space
             gs_seed = seed - 10 if (seed - 10) >= 0 else seed - 11 + (1 << 32)
             self._gs_seed = gs_seed
+            if self.lexico_objectives:
+                metric, mode = self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
+            self.init_lexicographic_obj()
             if experimental:
                 import optuna as ot
 
@@ -251,50 +242,27 @@ def __init__(
                     sampler = ot.samplers.MOTPESampler(
                         seed=gs_seed, n_startup_trials=n_startup_trials, n_ehvi_candidates=24
                     )
-
             else:
                 sampler = None
-            if self.lexico_objectives:
-                metric, mode = self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
             try:
                 assert evaluated_rewards
-                if self.lexico_objectives:
-                    self._gs = LexiGlobalSearch(
-                        space=gs_space,
-                        metric=metric,
-                        mode=mode,
-                        seed=gs_seed,
-                        sampler=sampler,
-                        points_to_evaluate=self._evaluated_points,
-                        evaluated_rewards=evaluated_rewards,
-                    )
-                else:
-                    self._gs = GlobalSearch(
-                        space=gs_space,
-                        metric=metric,
-                        mode=mode,
-                        seed=gs_seed,
-                        sampler=sampler,
-                        points_to_evaluate=self._evaluated_points,
-                        evaluated_rewards=evaluated_rewards,
-                    )
+                self._gs = GlobalSearch(
+                    space=gs_space,
+                    metric=metric,
+                    mode=mode,
+                    seed=gs_seed,
+                    sampler=sampler,
+                    points_to_evaluate=self._evaluated_points,
+                    evaluated_rewards=evaluated_rewards,
+                )
             except (AssertionError, ValueError):
-                if self.lexico_objectives:
-                    self._gs = LexiGlobalSearch(
-                        space=gs_space,
-                        metric=metric,
-                        mode=mode,
-                        seed=gs_seed,
-                        sampler=sampler,
-                    )
-                else:
-                    self._gs = GlobalSearch(
-                        space=gs_space,
-                        metric=metric,
-                        mode=mode,
-                        seed=gs_seed,
-                        sampler=sampler,
-                    )
+                self._gs = GlobalSearch(
+                    space=gs_space,
+                    metric=metric,
+                    mode=mode,
+                    seed=gs_seed,
+                    sampler=sampler,
+                )
             if isinstance(self._gs, LexiGlobalSearch):
                 self._gs.lexico_objectives = self.lexico_objectives
             self._gs.space = space
@@ -312,6 +280,20 @@ def __init__(
         if space is not None:
             self._init_search()
 
+    def init_lexicographic_obj(
+        self,
+    ):
+        self._f_best = None
+        self._histories = None
+        if self.lexico_objectives:
+            global GlobalSearch
+            GlobalSearch = LexiGlobalSearch
+            self._metric = self.lexico_objectives["metrics"][0]
+            self._mode = self.lexico_objectives["modes"][0]
+            if self._metric_constraints:
+                self._metric_constraints = None
+                logger.info("Do not support providing metric_constraints in lexicographic optimization for now.")
+
     def update_fbest(
         self,
     ):
@@ -373,20 +355,12 @@ def set_search_properties(
                 # reset search when metric or mode changed
                 self._ls.set_search_properties(metric, mode)
                 if self._gs is not None:
-                    if self.lexico_objectives:
-                        self._gs = LexiGlobalSearch(
-                            space=self._gs._space,
-                            metric=metric,
-                            mode=mode,
-                            seed=self._gs_seed,
-                        )
-                    else:
-                        self._gs = GlobalSearch(
-                            space=self._gs._space,
-                            metric=metric,
-                            mode=mode,
-                            seed=self._gs_seed,
-                        )
+                    self._gs = GlobalSearch(
+                        space=self._gs._space,
+                        metric=metric,
+                        mode=mode,
+                        seed=self._gs_seed,
+                    )
                     if isinstance(self._gs, LexiGlobalSearch):
                         self._gs.lexico_objectives = self.lexico_objective
                     self._gs.space = self._ls.space
@@ -956,19 +930,17 @@ def _violate_config_constriants(self, config, config_signature):
                 or sign == "<"
                 and value > threshold
             ):
-                if not self.lexico_objectives:
+                if self.lexico_objectives:
+                    for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
+                        self._result[config_signature] = {}
+                        self._result[config_signature][k_metric] = np.inf * -1 if k_mode == "max" else np.inf
+                        self._result[config_signature]["time_total_s"] = 1
+                else:
                     self._result[config_signature] = {
                         self._metric: np.inf * self._ls.metric_op,
                         "time_total_s": 1,
                     }
-                else:
-                    for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                        self._result[config_signature] = {}
-                        self._result[config_signature][k_metric] = {
-                            self._metric: np.inf * -1 if k_mode == "max" else np.inf
-                        }
-                        self._result[config_signature]["time_total_s"] = 1
-                return True
+                    return True
         return False
 
     def _should_skip(self, choice, trial_id, config, space) -> bool:
@@ -1224,20 +1196,14 @@ def update_search_space(self, search_space):
             lexico_objectives=self.lexico_objectives,
         )
         if self._gs is not None:
-            if self.lexico_objectives:
-                self._gs = LexiGlobalSearch(
-                    space=config,
-                    metric=self._metric,
-                    mode=self._mode,
-                    sampler=self._gs._sampler,
-                )
-            else:
-                self._gs = GlobalSearch(
-                    space=config,
-                    metric=self._metric,
-                    mode=self._mode,
-                    sampler=self._gs._sampler,
-                )
+            self._gs = GlobalSearch(
+                space=config,
+                metric=self._metric,
+                mode=self._mode,
+                sampler=self._gs._sampler,
+            )
+            if isinstance(self._gs, LexiGlobalSearch):
+                self._gs.lexico_objectives = self.lexico_objectives
             self._gs.space = config
         self._init_search()
 
diff --git a/flaml/tune/utils.py b/flaml/tune/utils.py
index 96b86ec27d..919f131ed4 100644
--- a/flaml/tune/utils.py
+++ b/flaml/tune/utils.py
@@ -28,6 +28,9 @@ def choice(categories: Sequence, order=None):
 
 
 def get_lexico_bound(metric, mode, lexico_objectives, f_best):
+    """Get targeted vector according to the historical points.
+    LexiFlow uses targeted vector to justify the order of different configurations.
+    """
     k_target = lexico_objectives["targets"][metric] if mode == "min" else -1 * lexico_objectives["targets"][metric]
     if not isinstance(lexico_objectives["tolerances"][metric], str):
         tolerance_bound = f_best[metric] + lexico_objectives["tolerances"][metric]

From 1ba9b902f48af549be8dba4daccfc647d85303cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Mon, 26 Jun 2023 09:52:41 +0800
Subject: [PATCH 12/15] fix lexiglobal sampler error

---
 flaml/tune/searcher/blendsearch.py |   5 +-
 flaml/tune/searcher/suggestion.py  | 105 +++++++++++++----------------
 2 files changed, 50 insertions(+), 60 deletions(-)

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index 93bc743416..386544c3c6 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -7,7 +7,7 @@
 import numpy as np
 import time
 import pickle
-
+from typing import Any
 
 try:
     from ray import __version__ as ray_version
@@ -22,7 +22,7 @@
 except (ImportError, AssertionError):
     from .suggestion import Searcher
     from .suggestion import OptunaSearch as NormalGlobalSearch
-from .suggestion import LexiGlobalSearch as LexiGlobalSearch
+from .suggestion import LexiGlobalSearch
 from ..trial import unflatten_dict, flatten_dict
 from .. import INCUMBENT_RESULT
 from .search_thread import SearchThread
@@ -30,7 +30,6 @@
 from ..space import add_cost_to_space, indexof, normalize, define_by_run_func
 from ..result import TIME_TOTAL_S
 from ..utils import get_lexico_bound
-
 import logging
 
 SEARCH_THREAD_EPS = 1.0
diff --git a/flaml/tune/searcher/suggestion.py b/flaml/tune/searcher/suggestion.py
index 2efccb94e8..e6917aae71 100644
--- a/flaml/tune/searcher/suggestion.py
+++ b/flaml/tune/searcher/suggestion.py
@@ -311,6 +311,7 @@ def validate_warmstart(
 
 class _OptunaTrialSuggestCaptor:
     """Utility to capture returned values from Optuna's suggest_ methods.
+
     This will wrap around the ``optuna.Trial` object and decorate all
     `suggest_` callables with a function capturing the returned value,
     which will be saved in the ``captured_values`` dict.
@@ -365,7 +366,7 @@ class OptunaSearch(Searcher):
             .. warning::
                 No actual computation should take place in the define-by-run
                 function. Instead, put the training logic inside the function
-                or class trainable passed to ``tune.Tuner()``.
+                or class trainable passed to ``tune.run``.
 
         metric: The training result objective value attribute. If
             None but a mode was passed, the anonymous metric ``_metric``
@@ -384,8 +385,6 @@ class OptunaSearch(Searcher):
             draw hyperparameter configurations. Defaults to ``MOTPESampler``
             for multi-objective optimization with Optuna<2.9.0, and
             ``TPESampler`` in every other case.
-            See https://optuna.readthedocs.io/en/stable/reference/samplers/index.html
-            for available Optuna samplers.
 
             .. warning::
                 Please note that with Optuna 2.10.0 and earlier
@@ -417,7 +416,7 @@ class OptunaSearch(Searcher):
 
     .. code-block:: python
 
-        from ray.tune.search.optuna import OptunaSearch
+        from ray.tune.suggest.optuna import OptunaSearch
 
         config = {
             "a": tune.uniform(6, 8)
@@ -428,21 +427,14 @@ class OptunaSearch(Searcher):
             metric="loss",
             mode="min")
 
-        tuner = tune.Tuner(
-            trainable,
-            tune_config=tune.TuneConfig(
-                search_alg=optuna_search,
-            ),
-            param_space=config,
-        )
-        tuner.fit()
+        tune.run(trainable, config=config, search_alg=optuna_search)
 
     If you would like to pass the search space manually, the code would
     look like this:
 
     .. code-block:: python
 
-        from ray.tune.search.optuna import OptunaSearch
+        from ray.tune.suggest.optuna import OptunaSearch
         import optuna
 
         space = {
@@ -455,13 +447,7 @@ class OptunaSearch(Searcher):
             metric="loss",
             mode="min")
 
-        tuner = tune.Tuner(
-            trainable,
-            tune_config=tune.TuneConfig(
-                search_alg=optuna_search,
-            ),
-        )
-        tuner.fit()
+        tune.run(trainable, search_alg=optuna_search)
 
         # Equivalent Optuna define-by-run function approach:
 
@@ -476,19 +462,13 @@ def define_search_space(trial: optuna.Trial):
             metric="loss",
             mode="min")
 
-        tuner = tune.Tuner(
-            trainable,
-            tune_config=tune.TuneConfig(
-                search_alg=optuna_search,
-            ),
-        )
-        tuner.fit()
+        tune.run(trainable, search_alg=optuna_search)
 
     Multi-objective optimization is supported:
 
     .. code-block:: python
 
-        from ray.tune.search.optuna import OptunaSearch
+        from ray.tune.suggest.optuna import OptunaSearch
         import optuna
 
         space = {
@@ -497,27 +477,24 @@ def define_search_space(trial: optuna.Trial):
         }
 
         # Note you have to specify metric and mode here instead of
-        # in tune.TuneConfig
+        # in tune.run
         optuna_search = OptunaSearch(
             space,
             metric=["loss1", "loss2"],
             mode=["min", "max"])
 
         # Do not specify metric and mode here!
-        tuner = tune.Tuner(
+        tune.run(
             trainable,
-            tune_config=tune.TuneConfig(
-                search_alg=optuna_search,
-            ),
+            search_alg=optuna_search
         )
-        tuner.fit()
 
     You can pass configs that will be evaluated first using
     ``points_to_evaluate``:
 
     .. code-block:: python
 
-        from ray.tune.search.optuna import OptunaSearch
+        from ray.tune.suggest.optuna import OptunaSearch
         import optuna
 
         space = {
@@ -531,20 +508,14 @@ def define_search_space(trial: optuna.Trial):
             metric="loss",
             mode="min")
 
-        tuner = tune.Tuner(
-            trainable,
-            tune_config=tune.TuneConfig(
-                search_alg=optuna_search,
-            ),
-        )
-        tuner.fit()
+        tune.run(trainable, search_alg=optuna_search)
 
     Avoid re-running evaluated trials by passing the rewards together with
     `points_to_evaluate`:
 
     .. code-block:: python
 
-        from ray.tune.search.optuna import OptunaSearch
+        from ray.tune.suggest.optuna import OptunaSearch
         import optuna
 
         space = {
@@ -559,13 +530,7 @@ def define_search_space(trial: optuna.Trial):
             metric="loss",
             mode="min")
 
-        tuner = tune.Tuner(
-            trainable,
-            tune_config=tune.TuneConfig(
-                search_alg=optuna_search,
-            ),
-        )
-        tuner.fit()
+        tune.run(trainable, search_alg=optuna_search)
 
     .. versionadded:: 0.8.8
 
@@ -707,7 +672,7 @@ def _suggest_from_define_by_run_func(
                 f"took {time_taken} seconds to "
                 "run. Ensure that actual computation, training takes "
                 "place inside Tune's train functions or Trainables "
-                "passed to `tune.Tuner()`."
+                "passed to `tune.run`."
             )
         if ret is not None:
             if not isinstance(ret, dict):
@@ -812,7 +777,7 @@ def add_evaluated_point(
                 "Define-by-run function passed in `space` argument is not "
                 "yet supported when using `evaluated_rewards`. Please provide "
                 "an `OptunaDistribution` dict or pass a Ray Tune "
-                "search space to `tune.Tuner()`."
+                "search space to `tune.run()`."
             )
 
         ot_trial_state = OptunaTrialState.COMPLETE
@@ -837,15 +802,27 @@ def add_evaluated_point(
         self._ot_study.add_trial(trial)
 
     def save(self, checkpoint_path: str):
-        save_object = self.__dict__.copy()
+        save_object = (
+            self._sampler,
+            self._ot_trials,
+            self._ot_study,
+            self._points_to_evaluate,
+            self._evaluated_rewards,
+        )
         with open(checkpoint_path, "wb") as outputFile:
             pickle.dump(save_object, outputFile)
 
     def restore(self, checkpoint_path: str):
         with open(checkpoint_path, "rb") as inputFile:
             save_object = pickle.load(inputFile)
-        if isinstance(save_object, dict):
-            self.__dict__.update(save_object)
+        if len(save_object) == 5:
+            (
+                self._sampler,
+                self._ot_trials,
+                self._ot_study,
+                self._points_to_evaluate,
+                self._evaluated_rewards,
+            ) = save_object
         else:
             # Backwards compatibility
             (
@@ -853,7 +830,6 @@ def restore(self, checkpoint_path: str):
                 self._ot_trials,
                 self._ot_study,
                 self._points_to_evaluate,
-                self._evaluated_rewards,
             ) = save_object
 
     @staticmethod
@@ -926,7 +902,22 @@ def resolve_value(domain: Domain) -> ot.distributions.BaseDistribution:
         return values
 
 
-class LexiGlobalSearch(OptunaSearch):
+try:
+    from ray import __version__ as ray_version
+
+    assert ray_version >= "1.10.0"
+    if ray_version.startswith("1."):
+        from ray.tune.suggest import Searcher as Searcher_inherited
+        from ray.tune.suggest.optuna import OptunaSearch as OptunaSearch_inherited
+    else:
+        from ray.tune.search import Searcher as Searcher_inherited
+        from ray.tune.search.optuna import OptunaSearch as OptunaSearch_inherited
+except (ImportError, AssertionError):
+    Searcher_inherited = Searcher
+    OptunaSearch_inherited = OptunaSearch
+
+
+class LexiGlobalSearch(OptunaSearch_inherited):
     def __init__(
         self,
         space: Optional[

From df9440ed27a4732599f25437d1de53de62a6a358 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Mon, 26 Jun 2023 11:01:30 +0800
Subject: [PATCH 13/15] follow chi's old suggestions

---
 flaml/tune/searcher/blendsearch.py   | 85 ++++++++++++----------------
 flaml/tune/searcher/flow2.py         | 26 ++++-----
 flaml/tune/searcher/search_thread.py |  4 +-
 flaml/tune/searcher/suggestion.py    | 22 +++----
 4 files changed, 61 insertions(+), 76 deletions(-)

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index 386544c3c6..ce3793bdb1 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -4,10 +4,10 @@
 #  * project root for license information.
 from typing import Dict, Optional, List, Tuple, Callable, Union
 from collections import defaultdict
+from functools import cmp_to_key
 import numpy as np
 import time
 import pickle
-from typing import Any
 
 try:
     from ray import __version__ as ray_version
@@ -165,6 +165,9 @@ def __init__(
             self.cost_attr = cost_attr
             self._cost_budget = cost_budget
         self.penalty = PENALTY  # penalty term for constraints
+        self._metric, self._mode = metric, mode
+        self.f_best = None
+        self.histories = None
         self._use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
         init_config = low_cost_partial_config or {}
         if not init_config:
@@ -196,10 +199,15 @@ def __init__(
             self._evaluated_rewards = evaluated_rewards or []
         self._config_constraints = config_constraints
         self._metric_constraints = metric_constraints
-        if metric_constraints and not self.lexico_objectives:
-            assert all(x[1] in ["<=", ">="] for x in metric_constraints), "sign of metric constraints must be <= or >=."
-            # metric modified by lagrange
-            metric += self.lagrange
+        if metric_constraints:
+            if self.lexico_objectives:
+                raise ValueError("Metric constraints should be provided via targets in lexicographic objectives.")
+            else:
+                assert all(
+                    x[1] in ["<=", ">="] for x in metric_constraints
+                ), "sign of metric constraints must be <= or >=."
+                # metric modified by lagrange
+                metric += self.lagrange
         self._cat_hp_cost = cat_hp_cost or {}
         if space:
             add_cost_to_space(space, init_config, self._cat_hp_cost)
@@ -230,7 +238,9 @@ def __init__(
             self._gs_seed = gs_seed
             if self.lexico_objectives:
                 metric, mode = self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
-            self.init_lexicographic_obj()
+            if self.lexico_objectives:
+                global GlobalSearch
+                GlobalSearch = LexiGlobalSearch
             if experimental:
                 import optuna as ot
 
@@ -279,36 +289,22 @@ def __init__(
         if space is not None:
             self._init_search()
 
-    def init_lexicographic_obj(
-        self,
-    ):
-        self._f_best = None
-        self._histories = None
-        if self.lexico_objectives:
-            global GlobalSearch
-            GlobalSearch = LexiGlobalSearch
-            self._metric = self.lexico_objectives["metrics"][0]
-            self._mode = self.lexico_objectives["modes"][0]
-            if self._metric_constraints:
-                self._metric_constraints = None
-                logger.info("Do not support providing metric_constraints in lexicographic optimization for now.")
-
     def update_fbest(
         self,
     ):
         obj_initial = self.lexico_objectives["metrics"][0]
-        feasible_index = np.array([*range(len(self._histories[obj_initial]))])
+        feasible_index = np.array([*range(len(self.histories[obj_initial]))])
         for k_metric in self.lexico_objectives["metrics"]:
-            k_values = np.array(self._histories[k_metric])
+            k_values = np.array(self.histories[k_metric])
             feasible_value = k_values.take(feasible_index)
-            self._f_best[k_metric] = np.min(feasible_value)
+            self.f_best[k_metric] = np.min(feasible_value)
             if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
-                tolerance_bound = self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
+                tolerance_bound = self.f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
             else:
                 assert (
                     self.lexico_objectives["tolerances"][k_metric][-1] == "%"
                 ), "String tolerance of {} should use %% as the suffix".format(k_metric)
-                tolerance_bound = self._f_best[k_metric] * (
+                tolerance_bound = self.f_best[k_metric] * (
                     1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))
                 )
             feasible_index_filter = np.where(
@@ -330,7 +326,7 @@ def set_search_properties(
         metric_changed = mode_changed = False
         if metric and self._metric != metric:
             metric_changed = True
-            self._metric = metric
+            # self._metric = metric
             if self._metric_constraints:
                 # metric modified by lagrange
                 metric += self.lagrange
@@ -499,10 +495,10 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
             del self._trial_proposed_by[trial_id]
         if result:
             if self.lexico_objectives:
-                if self._histories is None:
-                    self._histories, self._f_best = defaultdict(list), {}
+                if self.histories is None:
+                    self.histories, self.f_best = defaultdict(list), {}
                 for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                    self._histories[k_metric].append(result[k_metric] if k_mode == "min" else -1 * result[k_metric])
+                    self.histories[k_metric].append(result[k_metric] if k_mode == "min" else -1 * result[k_metric])
                 self.update_fbest()
             config = result.get("config", {})
             if not config:
@@ -634,7 +630,7 @@ def _create_condition(self, result: Dict) -> bool:
             return result[self._ls.metric] * self._ls.metric_op < obj_median
         else:
             thread_pools = [thread.obj_best1 for id, thread in self._search_thread_pool.items() if id]
-            thread_pools = self._lexico_sort(thread_pools)
+            thread_pools = sorted(thread_pools, key=cmp_to_key(self._lexico_inferior))
             obj_median = thread_pools[round(len(thread_pools) / 2)]
             result = self._unify_op(result)
             return self._lexico_inferior(obj_median, result)
@@ -720,7 +716,7 @@ def _expand_admissible_region(self, lower, upper, space):
     def _lexico_inferior(self, obj_1: Union[dict, float], obj_2: Union[dict, float]) -> bool:
         if self.lexico_objectives:
             for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                bound = get_lexico_bound(k_metric, k_mode, self.lexico_objectives, self._f_best)
+                bound = get_lexico_bound(k_metric, k_mode, self.lexico_objectives, self.f_best)
                 if (obj_1[k_metric] < bound) and (obj_2[k_metric] < bound):
                     continue
                 elif obj_1[k_metric] < obj_2[k_metric]:
@@ -735,10 +731,7 @@ def _lexico_inferior(self, obj_1: Union[dict, float], obj_2: Union[dict, float])
                 else:
                     return True
         else:
-            if obj_1 > obj_2:
-                return True
-            else:
-                return False
+            return obj_1 > obj_2
 
     def _unify_op(self, result: Union[dict, float]):
         if isinstance(result, dict):
@@ -748,14 +741,6 @@ def _unify_op(self, result: Union[dict, float]):
             result[self._ls.metric] = result[self._ls.metric] * self._ls.metric_op
         return result
 
-    def _lexico_sort(self, arr: list):
-        n = len(arr)
-        for i in range(n):
-            for j in range(0, n - i - 1):
-                if self._lexico_(arr[j], arr[j + 1]):
-                    arr[j], arr[j + 1] = arr[j + 1], arr[j]
-        return arr
-
     def _inferior(self, id1: int, id2: int) -> bool:
         """whether thread id1 is inferior to id2"""
         t1 = self._search_thread_pool[id1]
@@ -778,10 +763,10 @@ def on_trial_result(self, trial_id: str, result: Dict):
         if result and self._metric_constraints:
             result[self._metric + self.lagrange] = result[self._metric]
             if self.lexico_objectives:
-                if self._histories is None:
-                    self._histories, self._f_best = defaultdict(list), {}
+                if self.histories is None:
+                    self.histories, self.f_best = defaultdict(list), {}
                 for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                    self._histories[k_metric].append(result[k_metric] if k_mode == "min" else -1 * result[k_metric])
+                    self.histories[k_metric].append(result[k_metric] if k_mode == "min" else -1 * result[k_metric])
                 self.update_fbest()
         self._search_thread_pool[thread_id].on_trial_result(trial_id, result)
 
@@ -868,8 +853,8 @@ def suggest(self, trial_id: str) -> Optional[Dict]:
         else:  # use init config
             if self._candidate_start_points is not None and self._points_to_evaluate:
                 if self.lexico_objectives:
-                    raise NotImplementedError(
-                        "It doesn't support providing points_to_evaluate in lexicographic optimization for now."
+                    raise ValueError(
+                        "Providing points_to_evaluate in lexicographic optimization is not supported for now."
                     )
                 self._candidate_start_points[trial_id] = None
             reward = None
@@ -933,7 +918,7 @@ def _violate_config_constriants(self, config, config_signature):
                     for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
                         self._result[config_signature] = {}
                         self._result[config_signature][k_metric] = np.inf * -1 if k_mode == "max" else np.inf
-                        self._result[config_signature]["time_total_s"] = 1
+                    self._result[config_signature]["time_total_s"] = 1
                 else:
                     self._result[config_signature] = {
                         self._metric: np.inf * self._ls.metric_op,
@@ -1018,7 +1003,7 @@ def _select_thread(self) -> Tuple:
             else:
                 _metric_1st = self.lexico_objectives["metrics"][0]
                 _op_1st = self.lexico_objectives["modes"][0]
-                _lexico_target = self._f_best[_metric_1st] if _op_1st == "min" else -1 * self._f_best[_metric_1st]
+                _lexico_target = self.f_best[_metric_1st] if _op_1st == "min" else -1 * self.f_best[_metric_1st]
                 thread.update_eci(_lexico_target, max_speed, min_speed)
             if thread.eci < min_eci:
                 min_eci = thread.eci
diff --git a/flaml/tune/searcher/flow2.py b/flaml/tune/searcher/flow2.py
index 1aa44b5c8b..187aa63cf3 100644
--- a/flaml/tune/searcher/flow2.py
+++ b/flaml/tune/searcher/flow2.py
@@ -134,10 +134,10 @@ def __init__(
         self.cost_attr = cost_attr
         self.max_resource = max_resource
         self._resource = None
-        self._f_best = None  # only use for lexico_comapre. It represent the best value achieved by lexico_flow.
+        self.f_best = None  # only use for lexico_comapre. It represent the best value achieved by lexico_flow.
         self.op_dimension = None
         self._step_lb = np.Inf
-        self._histories = None  # only use for lexico_comapre. It records the result of historical configurations.
+        self.histories = None  # only use for lexico_comapre. It records the result of historical configurations.
         if space is not None:
             self._init_search()
 
@@ -338,18 +338,18 @@ def update_fbest(
         self,
     ):
         obj_initial = self.lexico_objectives["metrics"][0]
-        feasible_index = np.array([*range(len(self._histories[obj_initial]))])
+        feasible_index = np.array([*range(len(self.histories[obj_initial]))])
         for k_metric in self.lexico_objectives["metrics"]:
-            k_values = np.array(self._histories[k_metric])
+            k_values = np.array(self.histories[k_metric])
             feasible_value = k_values.take(feasible_index)
-            self._f_best[k_metric] = np.min(feasible_value)
+            self.f_best[k_metric] = np.min(feasible_value)
             if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
-                tolerance_bound = self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
+                tolerance_bound = self.f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
             else:
                 assert (
                     self.lexico_objectives["tolerances"][k_metric][-1] == "%"
                 ), "String tolerance of {} should use %% as the suffix".format(k_metric)
-                tolerance_bound = self._f_best[k_metric] * (
+                tolerance_bound = self.f_best[k_metric] * (
                     1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))
                 )
             feasible_index_filter = np.where(
@@ -362,15 +362,15 @@ def update_fbest(
             feasible_index = feasible_index.take(feasible_index_filter)
 
     def lexico_compare(self, result) -> bool:
-        if self._histories is None:
-            self._histories, self._f_best = defaultdict(list), {}
+        if self.histories is None:
+            self.histories, self.f_best = defaultdict(list), {}
             for k in self.lexico_objectives["metrics"]:
-                self._histories[k].append(result[k])
+                self.histories[k].append(result[k])
             self.update_fbest()
             return True
         else:
             for k in self.lexico_objectives["metrics"]:
-                self._histories[k].append(result[k])
+                self.histories[k].append(result[k])
             self.update_fbest()
             for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
                 k_target = (
@@ -379,12 +379,12 @@ def lexico_compare(self, result) -> bool:
                     else -self.lexico_objectives["targets"][k_metric]
                 )
                 if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
-                    tolerance_bound = self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
+                    tolerance_bound = self.f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
                 else:
                     assert (
                         self.lexico_objectives["tolerances"][k_metric][-1] == "%"
                     ), "String tolerance of {} should use %% as the suffix".format(k_metric)
-                    tolerance_bound = self._f_best[k_metric] * (
+                    tolerance_bound = self.f_best[k_metric] * (
                         1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))
                     )
                 if (result[k_metric] < max(tolerance_bound, k_target)) and (
diff --git a/flaml/tune/searcher/search_thread.py b/flaml/tune/searcher/search_thread.py
index a3cfabd110..95e6c69a50 100644
--- a/flaml/tune/searcher/search_thread.py
+++ b/flaml/tune/searcher/search_thread.py
@@ -123,7 +123,7 @@ def update_eci(self, metric_target: float, max_speed: Optional[float] = np.inf,
     def _better(self, obj_1: Union[dict, float], obj_2: Union[dict, float]):
         if self.lexico_objectives:
             for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                bound = get_lexico_bound(k_metric, k_mode, self.lexico_objectives, self._search_alg._f_best)
+                bound = get_lexico_bound(k_metric, k_mode, self.lexico_objectives, self._search_alg.f_best)
                 if (obj_1[k_metric] < bound) and (obj_2[k_metric] < bound):
                     continue
                 elif obj_1[k_metric] < obj_2[k_metric]:
@@ -154,7 +154,7 @@ def _update_speed(self):
                 )
             else:
                 self.speed = 0
-        elif self._search_alg._histories:
+        elif self._search_alg.histories:
             compare_tuple = self._better(self.obj_best1, self.obj_best2)
             if compare_tuple[0]:
                 if self._is_ls:
diff --git a/flaml/tune/searcher/suggestion.py b/flaml/tune/searcher/suggestion.py
index e6917aae71..3617dbbb84 100644
--- a/flaml/tune/searcher/suggestion.py
+++ b/flaml/tune/searcher/suggestion.py
@@ -927,16 +927,16 @@ def __init__(
                 Callable[["OptunaTrial"], Optional[Dict[str, Any]]],
             ]
         ] = None,
-        metric: Optional[str] = None,
-        mode: Optional[str] = None,
+        metric: Optional[Union[str, List[str]]] = None,
+        mode: Optional[Union[str, List[str]]] = None,
         points_to_evaluate: Optional[List[Dict]] = None,
         sampler: Optional["BaseSampler"] = None,
         seed: Optional[int] = None,
         evaluated_rewards: Optional[List] = None,
     ):
         super().__init__(space, metric, mode, points_to_evaluate, sampler, seed, evaluated_rewards)
-        self._f_best = None  # only use for lexico_comapre. It represent the best value achieved by the algorithm.
-        self._histories = None  # only use for lexico_comapre. It records the result of historical configurations.
+        self.f_best = None  # only use for lexico_comapre. It represent the best value achieved by the algorithm.
+        self.histories = None  # only use for lexico_comapre. It records the result of historical configurations.
 
     def set_search_properties(
         self, metric: Optional[Union[str, List[str]]], mode: Optional[Union[str, List[str]]], config: Dict
@@ -956,18 +956,18 @@ def update_fbest(
         self,
     ):
         obj_initial = self.lexico_objectives["metrics"][0]
-        feasible_index = np.array([*range(len(self._histories[obj_initial]))])
+        feasible_index = np.array([*range(len(self.histories[obj_initial]))])
         for k_metric in self.lexico_objectives["metrics"]:
-            k_values = np.array(self._histories[k_metric])
+            k_values = np.array(self.histories[k_metric])
             feasible_value = k_values.take(feasible_index)
-            self._f_best[k_metric] = np.min(feasible_value)
+            self.f_best[k_metric] = np.min(feasible_value)
             if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
-                tolerance_bound = self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
+                tolerance_bound = self.f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
             else:
                 assert (
                     self.lexico_objectives["tolerances"][k_metric][-1] == "%"
                 ), "String tolerance of {} should use %% as the suffix".format(k_metric)
-                tolerance_bound = self._f_best[k_metric] * (
+                tolerance_bound = self.f_best[k_metric] * (
                     1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))
                 )
             feasible_index_filter = np.where(
@@ -982,7 +982,7 @@ def update_fbest(
     def on_trial_result(self, trial_id: str, result: Dict):
         super.on_trial_result(trial_id, result)
         for k_metric, k_mode in zip(self.lexico_objectives["metrics"]):
-            self._histories[k_metric].append(result[k_metric]) if k_mode == "min" else self._histories[k_metric].append(
+            self.histories[k_metric].append(result[k_metric]) if k_mode == "min" else self.histories[k_metric].append(
                 result[k_metric] * -1
             )
         self.update_fbest()
@@ -990,7 +990,7 @@ def on_trial_result(self, trial_id: str, result: Dict):
     def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
         super.on_trial_complete(trial_id, result, error)
         for k_metric, k_mode in zip(self.lexico_objectives["metrics"]):
-            self._histories[k_metric].append(result[k_metric]) if k_mode == "min" else self._histories[k_metric].append(
+            self.histories[k_metric].append(result[k_metric]) if k_mode == "min" else self.histories[k_metric].append(
                 result[k_metric] * -1
             )
         self.update_fbest()

From 5969d165cea182e5d148804124bc016ec0f67d0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Mon, 26 Jun 2023 16:55:17 +0800
Subject: [PATCH 14/15] remove lexicoglobalsearch

---
 flaml/tune/searcher/blendsearch.py   |  28 +++---
 flaml/tune/searcher/search_thread.py | 125 +++++++++++++++++----------
 flaml/tune/searcher/suggestion.py    |  95 +-------------------
 test/tune/test_lexiflow.py           |  21 ++++-
 4 files changed, 113 insertions(+), 156 deletions(-)

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index ce3793bdb1..f2db25048e 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -15,14 +15,13 @@
     assert ray_version >= "1.10.0"
     if ray_version.startswith("1."):
         from ray.tune.suggest import Searcher
-        from ray.tune.suggest.optuna import OptunaSearch as NormalGlobalSearch
+        from ray.tune.suggest.optuna import OptunaSearch as GlobalSearch
     else:
         from ray.tune.search import Searcher
-        from ray.tune.search.optuna import OptunaSearch as NormalGlobalSearch
+        from ray.tune.search.optuna import OptunaSearch as GlobalSearch
 except (ImportError, AssertionError):
     from .suggestion import Searcher
-    from .suggestion import OptunaSearch as NormalGlobalSearch
-from .suggestion import LexiGlobalSearch
+    from .suggestion import OptunaSearch as GlobalSearch
 from ..trial import unflatten_dict, flatten_dict
 from .. import INCUMBENT_RESULT
 from .search_thread import SearchThread
@@ -35,7 +34,6 @@
 SEARCH_THREAD_EPS = 1.0
 PENALTY = 1e10  # penalty term for constraints
 logger = logging.getLogger(__name__)
-GlobalSearch = NormalGlobalSearch
 
 
 class BlendSearch(Searcher):
@@ -238,9 +236,6 @@ def __init__(
             self._gs_seed = gs_seed
             if self.lexico_objectives:
                 metric, mode = self.lexico_objectives["metrics"], self.lexico_objectives["modes"]
-            if self.lexico_objectives:
-                global GlobalSearch
-                GlobalSearch = LexiGlobalSearch
             if experimental:
                 import optuna as ot
 
@@ -272,7 +267,7 @@ def __init__(
                     seed=gs_seed,
                     sampler=sampler,
                 )
-            if isinstance(self._gs, LexiGlobalSearch):
+            if self.lexico_objectives:
                 self._gs.lexico_objectives = self.lexico_objectives
             self._gs.space = space
         else:
@@ -356,7 +351,7 @@ def set_search_properties(
                         mode=mode,
                         seed=self._gs_seed,
                     )
-                    if isinstance(self._gs, LexiGlobalSearch):
+                    if self.lexico_objectives:
                         self._gs.lexico_objectives = self.lexico_objective
                     self._gs.space = self._ls.space
                 self._init_search()
@@ -629,8 +624,10 @@ def _create_condition(self, result: Dict) -> bool:
             obj_median = np.median([thread.obj_best1 for id, thread in self._search_thread_pool.items() if id])
             return result[self._ls.metric] * self._ls.metric_op < obj_median
         else:
-            thread_pools = [thread.obj_best1 for id, thread in self._search_thread_pool.items() if id]
-            thread_pools = sorted(thread_pools, key=cmp_to_key(self._lexico_inferior))
+            thread_pools = sorted(
+                [thread.obj_best1 for id, thread in self._search_thread_pool.items() if id],
+                key=cmp_to_key(self._lexico_inferior),
+            )
             obj_median = thread_pools[round(len(thread_pools) / 2)]
             result = self._unify_op(result)
             return self._lexico_inferior(obj_median, result)
@@ -989,8 +986,9 @@ def _select_thread(self) -> Tuple:
                 if thread.speed < min_speed:
                     min_speed = thread.speed
         else:
-            max_speed = {k: 0 for k in self.lexico_objectives["metrics"]}
-            min_speed = {k: float("inf") for k in self.lexico_objectives["metrics"]}
+            max_speed, min_speed = {k: 0 for k in self.lexico_objectives["metrics"]}, {
+                k: float("inf") for k in self.lexico_objectives["metrics"]
+            }
             for k_metric in self.lexico_objectives["metrics"]:
                 for thread in self._search_thread_pool.values():
                     if thread.speed[k_metric] > max_speed[k_metric]:
@@ -1186,7 +1184,7 @@ def update_search_space(self, search_space):
                 mode=self._mode,
                 sampler=self._gs._sampler,
             )
-            if isinstance(self._gs, LexiGlobalSearch):
+            if self.lexico_objectives:
                 self._gs.lexico_objectives = self.lexico_objectives
             self._gs.space = config
         self._init_search()
diff --git a/flaml/tune/searcher/search_thread.py b/flaml/tune/searcher/search_thread.py
index 95e6c69a50..9bc7e5c705 100644
--- a/flaml/tune/searcher/search_thread.py
+++ b/flaml/tune/searcher/search_thread.py
@@ -21,6 +21,7 @@
 from ..result import TIME_TOTAL_S
 from ..utils import get_lexico_bound
 import logging
+from collections import defaultdict
 
 logger = logging.getLogger(__name__)
 
@@ -44,34 +45,34 @@ def __init__(
         self._eps = eps
         self.cost_best2 = 0
         self.lexico_objectives = getattr(self._search_alg, "lexico_objectives", None)
+        self.best_result = None
+        # eci: estimated cost for improvement
+        self.eci = self.cost_best
+        self._init_config = True
+        self.running = 0  # the number of running trials from the thread
+        self.cost_attr = cost_attr
+        if search_alg:
+            self.space = self._space = search_alg.space  # unflattened space
+            if self.space and not isinstance(search_alg, FLOW2) and isinstance(search_alg._space, dict):
+                # remember const config
+                self._const = add_cost_to_space(self.space, {}, {})
 
         if self.lexico_objectives:
-            # set 1st to 0 others to -1?
+            # lexicographic tuning setting
             self.obj_best1 = self.obj_best2 = {}
             for k_metric in self.lexico_objectives["metrics"]:
                 self.obj_best1[k_metric] = self.obj_best2[k_metric] = (
                     np.inf if getattr(search_alg, "best_obj", None) is None else search_alg.best_obj[k_metric]
                 )
-        else:
-            self.obj_best1 = self.obj_best2 = getattr(search_alg, "best_obj", np.inf)  # inherently minimize
-
-        self.best_result = None
-        # eci: estimated cost for improvement
-        self.eci = self.cost_best
-        if self.lexico_objectives:
             self.priority, self.speed = {}, {}
             for k_metric in self.lexico_objectives["metrics"]:
                 self.priority[k_metric] = self.speed[k_metric] = 0
         else:
+            # normal tuning setting
+            self.obj_best1 = self.obj_best2 = getattr(search_alg, "best_obj", np.inf)  # inherently minimize
             self.priority = self.speed = 0
-        self._init_config = True
-        self.running = 0  # the number of running trials from the thread
-        self.cost_attr = cost_attr
-        if search_alg:
-            self.space = self._space = search_alg.space  # unflattened space
-            if self.space and not isinstance(search_alg, FLOW2) and isinstance(search_alg._space, dict):
-                # remember const config
-                self._const = add_cost_to_space(self.space, {}, {})
+        if self.lexico_objectives:
+            self.f_best, self.histories = {}, defaultdict(list)  # only use for lexico_comapre.
 
     def suggest(self, trial_id: str) -> Optional[Dict]:
         """Use the suggest() of the underlying search algorithm."""
@@ -92,6 +93,36 @@ def suggest(self, trial_id: str) -> Optional[Dict]:
             self.running += 1
         return config
 
+    def update_lexicoinfo(self, result):
+        if self.lexico_objectives:
+            for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
+                self.histories[k_metric].append(result[k_metric]) if k_mode == "min" else self.histories[
+                    k_metric
+                ].append(result[k_metric] * -1)
+            obj_initial = self.lexico_objectives["metrics"][0]
+            feasible_index = np.array([*range(len(self.histories[obj_initial]))])
+            for k_metric in self.lexico_objectives["metrics"]:
+                k_values = np.array(self.histories[k_metric])
+                feasible_value = k_values.take(feasible_index)
+                self.f_best[k_metric] = np.min(feasible_value)
+                if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
+                    tolerance_bound = self.f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
+                else:
+                    assert (
+                        self.lexico_objectives["tolerances"][k_metric][-1] == "%"
+                    ), "String tolerance of {} should use %% as the suffix".format(k_metric)
+                    tolerance_bound = self.f_best[k_metric] * (
+                        1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))
+                    )
+                feasible_index_filter = np.where(
+                    feasible_value
+                    <= max(
+                        tolerance_bound,
+                        self.lexico_objectives["targets"][k_metric],
+                    )
+                )[0]
+                feasible_index = feasible_index.take(feasible_index_filter)
+
     def update_priority(self, eci: Optional[float] = 0):
         # optimistic projection
         if self.lexico_objectives:
@@ -110,43 +141,45 @@ def update_eci(self, metric_target: float, max_speed: Optional[float] = np.inf,
             _metric_1st = self.lexico_objectives["metrics"][0]
             _metric_op = 1 if self.lexico_objectives["modes"][0] == "min" else -1
             if self.speed[_metric_1st] == 0:
-                self.speed = max_speed
+                self.speed[_metric_1st] = max_speed[_metric_1st]
             elif self.speed[_metric_1st] == -1:
-                self.speed = min_speed
+                self.speed[_metric_1st] = min_speed[_metric_1st]
         best_obj = metric_target * _metric_op
         self.eci = max(self.cost_total - self.cost_best1, self.cost_best1 - self.cost_best2)
         obj_best1 = self.obj_best1 if not self.lexico_objectives else self.obj_best1[_metric_1st]
         speed = self.speed if not self.lexico_objectives else self.speed[_metric_1st]
         if obj_best1 > best_obj and speed > 0:
-            self.eci = max(self.eci, 2 * (self.obj_best1 - best_obj) / self.speed)
+            self.eci = max(self.eci, 2 * (obj_best1 - best_obj) / speed)
 
     def _better(self, obj_1: Union[dict, float], obj_2: Union[dict, float]):
         if self.lexico_objectives:
             for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
-                bound = get_lexico_bound(k_metric, k_mode, self.lexico_objectives, self._search_alg.f_best)
+                _f_best = self._search_alg.f_best if self._is_ls else self.f_best
+                bound = get_lexico_bound(k_metric, k_mode, self.lexico_objectives, _f_best)
                 if (obj_1[k_metric] < bound) and (obj_2[k_metric] < bound):
                     continue
                 elif obj_1[k_metric] < obj_2[k_metric]:
-                    return (True, k_metric)
+                    return True, k_metric
                 else:
-                    return (False, None)
+                    return False, None
             for k_metr in self.lexico_objectives["metrics"]:
                 if obj_1[k_metr] == obj_2[k_metr]:
                     continue
                 elif obj_1[k_metr] < obj_2[k_metr]:
-                    return (True, k_metric)
+                    return True, k_metric
                 else:
-                    return (False, None)
+                    return False, None
+            return False, None
         else:
             if obj_1 < obj_2:
-                return True
+                return True, None
             else:
-                return False
+                return False, None
 
     def _update_speed(self):
         # calculate speed; use 0 for invalid speed temporarily
         if not self.lexico_objectives:
-            if self._better(self.obj_best1, self.obj_best2):
+            if self.obj_best1 < self.obj_best2:
                 self.speed = (
                     (self.obj_best2 - self.obj_best1)
                     / self.running
@@ -154,21 +187,16 @@ def _update_speed(self):
                 )
             else:
                 self.speed = 0
-        elif self._search_alg.histories:
-            compare_tuple = self._better(self.obj_best1, self.obj_best2)
-            if compare_tuple[0]:
-                if self._is_ls:
-                    op_dimension = self._search_alg.op_dimension
-                else:
-                    op_dimension = compare_tuple[1]
-                op_index = self.lexico_objectives["metrics"].index(op_dimension)
-                metrics_length = len(self.lexico_objectives["metrics"])
-                self.speed[op_dimension] = (
-                    (self.obj_best2[op_dimension] - self.obj_best1[op_dimension])
+        elif (self._is_ls and self._search_alg.histories) or (not self._is_ls and self.histories):
+            _is_better, _op_dimension = self._better(self.obj_best1, self.obj_best2)
+            if _is_better:
+                op_index = self.lexico_objectives["metrics"].index(_op_dimension)
+                self.speed[_op_dimension] = (
+                    (self.obj_best2[_op_dimension] - self.obj_best1[_op_dimension])
                     / self.running
                     / (max(self.cost_total - self.cost_best2, self._eps))
                 )
-                for i in range(0, metrics_length):
+                for i in range(0, len(self.lexico_objectives["metrics"])):
                     if i < op_index:
                         self.speed[self.lexico_objectives["metrics"][i]] = -1
                     elif i > op_index:
@@ -176,6 +204,8 @@ def _update_speed(self):
             else:
                 for k_metric in self.lexico_objectives["metrics"]:
                     self.speed[k_metric] = 0
+        else:
+            return
 
     def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
         """Update the statistics of the thread."""
@@ -186,6 +216,8 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
             if self._is_ls or not self._init_config:
                 try:
                     self._search_alg.on_trial_complete(trial_id, result, error)
+                    if not self._is_ls:
+                        self.update_lexicoinfo(result)
                 except RuntimeError as e:
                     # rs is used in place of optuna sometimes
                     if not str(e).endswith("has already finished and can not be updated."):
@@ -197,11 +229,12 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
         if result:
             self.cost_last = result.get(self.cost_attr, 1)
             self.cost_total += self.cost_last
-            if not self.lexico_objectives:
-                feasible_condition = self._search_alg.metric in result
-            else:
-                feasible_condition = all(x in result for x in self._search_alg.metric)
-            if feasible_condition:
+            _metric_exists = (
+                self._search_alg.metric in result
+                if not self.lexico_objectives
+                else all(x in result for x in self.lexico_objectives["metrics"])
+            )
+            if _metric_exists:
                 if not self.lexico_objectives:
                     obj = result[self._search_alg.metric] * self._metric_op
                 else:
@@ -210,7 +243,7 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
                         self._search_alg.lexico_objectives["metrics"], self._search_alg.lexico_objectives["modes"]
                     ):
                         obj[k] = -1 * result[k] if m == "max" else result[k]
-                if self.best_result is None or self._better(obj, self.obj_best1):
+                if self.best_result is None or self._better(obj, self.obj_best1)[0]:
                     self.cost_best2 = self.cost_best1
                     self.cost_best1 = self.cost_total
                     if not self.lexico_objectives:
@@ -233,6 +266,8 @@ def on_trial_result(self, trial_id: str, result: Dict):
         if not hasattr(self._search_alg, "_ot_trials") or (trial_id in self._search_alg._ot_trials):
             try:
                 self._search_alg.on_trial_result(trial_id, result)
+                if not self._is_ls:
+                    self.update_lexicoinfo(result)
             except RuntimeError as e:
                 # rs is used in place of optuna sometimes
                 if not str(e).endswith("has already finished and can not be updated."):
diff --git a/flaml/tune/searcher/suggestion.py b/flaml/tune/searcher/suggestion.py
index 3617dbbb84..57758a3ce1 100644
--- a/flaml/tune/searcher/suggestion.py
+++ b/flaml/tune/searcher/suggestion.py
@@ -35,6 +35,7 @@
 )
 from ..trial import flatten_dict, unflatten_dict
 from packaging import version
+from collections import defaultdict
 
 logger = logging.getLogger(__name__)
 
@@ -900,97 +901,3 @@ def resolve_value(domain: Domain) -> ot.distributions.BaseDistribution:
         values = {"/".join(path): resolve_value(domain) for path, domain in domain_vars}
 
         return values
-
-
-try:
-    from ray import __version__ as ray_version
-
-    assert ray_version >= "1.10.0"
-    if ray_version.startswith("1."):
-        from ray.tune.suggest import Searcher as Searcher_inherited
-        from ray.tune.suggest.optuna import OptunaSearch as OptunaSearch_inherited
-    else:
-        from ray.tune.search import Searcher as Searcher_inherited
-        from ray.tune.search.optuna import OptunaSearch as OptunaSearch_inherited
-except (ImportError, AssertionError):
-    Searcher_inherited = Searcher
-    OptunaSearch_inherited = OptunaSearch
-
-
-class LexiGlobalSearch(OptunaSearch_inherited):
-    def __init__(
-        self,
-        space: Optional[
-            Union[
-                Dict[str, "OptunaDistribution"],
-                List[Tuple],
-                Callable[["OptunaTrial"], Optional[Dict[str, Any]]],
-            ]
-        ] = None,
-        metric: Optional[Union[str, List[str]]] = None,
-        mode: Optional[Union[str, List[str]]] = None,
-        points_to_evaluate: Optional[List[Dict]] = None,
-        sampler: Optional["BaseSampler"] = None,
-        seed: Optional[int] = None,
-        evaluated_rewards: Optional[List] = None,
-    ):
-        super().__init__(space, metric, mode, points_to_evaluate, sampler, seed, evaluated_rewards)
-        self.f_best = None  # only use for lexico_comapre. It represent the best value achieved by the algorithm.
-        self.histories = None  # only use for lexico_comapre. It records the result of historical configurations.
-
-    def set_search_properties(
-        self, metric: Optional[Union[str, List[str]]], mode: Optional[Union[str, List[str]]], config: Dict
-    ) -> bool:
-        if self._space:
-            return False
-        space = self.convert_search_space(config)
-        self._space = space
-        if metric:
-            self._metric = metric
-        if mode:
-            self._mode = mode
-        self._setup_study(mode)
-        return True
-
-    def update_fbest(
-        self,
-    ):
-        obj_initial = self.lexico_objectives["metrics"][0]
-        feasible_index = np.array([*range(len(self.histories[obj_initial]))])
-        for k_metric in self.lexico_objectives["metrics"]:
-            k_values = np.array(self.histories[k_metric])
-            feasible_value = k_values.take(feasible_index)
-            self.f_best[k_metric] = np.min(feasible_value)
-            if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
-                tolerance_bound = self.f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
-            else:
-                assert (
-                    self.lexico_objectives["tolerances"][k_metric][-1] == "%"
-                ), "String tolerance of {} should use %% as the suffix".format(k_metric)
-                tolerance_bound = self.f_best[k_metric] * (
-                    1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))
-                )
-            feasible_index_filter = np.where(
-                feasible_value
-                <= max(
-                    tolerance_bound,
-                    self.lexico_objectives["targets"][k_metric],
-                )
-            )[0]
-            feasible_index = feasible_index.take(feasible_index_filter)
-
-    def on_trial_result(self, trial_id: str, result: Dict):
-        super.on_trial_result(trial_id, result)
-        for k_metric, k_mode in zip(self.lexico_objectives["metrics"]):
-            self.histories[k_metric].append(result[k_metric]) if k_mode == "min" else self.histories[k_metric].append(
-                result[k_metric] * -1
-            )
-        self.update_fbest()
-
-    def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
-        super.on_trial_complete(trial_id, result, error)
-        for k_metric, k_mode in zip(self.lexico_objectives["metrics"]):
-            self.histories[k_metric].append(result[k_metric]) if k_mode == "min" else self.histories[k_metric].append(
-                result[k_metric] * -1
-            )
-        self.update_fbest()
diff --git a/test/tune/test_lexiflow.py b/test/tune/test_lexiflow.py
index f5bb12bf66..974f095190 100644
--- a/test/tune/test_lexiflow.py
+++ b/test/tune/test_lexiflow.py
@@ -103,7 +103,7 @@ def evaluate_function(configuration):
 
     lexico_objectives = {}
     lexico_objectives["metrics"] = ["error_rate", "flops"]
-    lexico_objectives["algorithm"] = ["CFO"]
+    lexico_objectives["lexico_algorithm"] = "CFO"
 
     search_space = {
         "n_layers": tune.randint(lower=1, upper=3),
@@ -146,6 +146,7 @@ def evaluate_function(configuration):
 
     # 1. lexico tune: absolute tolerance
     lexico_objectives["tolerances"] = {"error_rate": 0.02, "flops": 0.0}
+    lexico_objectives["lexico_algorithm"] = "CFO"
     analysis = tune.run(
         evaluate_function,
         num_samples=5,
@@ -160,6 +161,22 @@ def evaluate_function(configuration):
 
     # 2. lexico tune: percentage tolerance
     lexico_objectives["tolerances"] = {"error_rate": "10%", "flops": "0%"}
+    lexico_objectives["lexico_algorithm"] = "CFO"
+    analysis = tune.run(
+        evaluate_function,
+        num_samples=5,
+        config=search_space,
+        use_ray=False,
+        lexico_objectives=lexico_objectives,
+        low_cost_partial_config=low_cost_partial_config,
+    )
+    print(analysis.best_trial)
+    print(analysis.best_config)
+    print(analysis.best_result)
+
+    # 3. lexico tune - blendsearch: percentage tolerance
+    lexico_objectives["tolerances"] = {"error_rate": "10%", "flops": "0%"}
+    lexico_objectives["lexico_algorithm"] = "BlendSearch"
     analysis = tune.run(
         evaluate_function,
         num_samples=5,
@@ -179,7 +196,7 @@ def test_lexiflow_performance():
     lexico_objectives["tolerances"] = {"brain": 10.0, "currin": 0.0}
     lexico_objectives["targets"] = {"brain": 0.0, "currin": 0.0}
     lexico_objectives["modes"] = ["min", "min"]
-    lexico_objectives["algorithm"] = ["CFO"]
+    lexico_objectives["lexico_algorithm"] = "CFO"
 
     search_space = {
         "x1": tune.uniform(lower=0.000001, upper=1.0),

From bb87ea547517a44d18b9822ca3d0614140074836 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cskzhang1=E2=80=9D?=
 <“shaokunzhang529@gmail.com”>
Date: Mon, 26 Jun 2023 17:13:19 +0800
Subject: [PATCH 15/15] clean up

---
 flaml/tune/searcher/blendsearch.py   |  2 +-
 flaml/tune/searcher/flow2.py         |  3 ---
 flaml/tune/searcher/search_thread.py | 10 +++++-----
 3 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/flaml/tune/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
index f2db25048e..f76b276f56 100644
--- a/flaml/tune/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -321,7 +321,7 @@ def set_search_properties(
         metric_changed = mode_changed = False
         if metric and self._metric != metric:
             metric_changed = True
-            # self._metric = metric
+            self._metric = metric
             if self._metric_constraints:
                 # metric modified by lagrange
                 metric += self.lagrange
diff --git a/flaml/tune/searcher/flow2.py b/flaml/tune/searcher/flow2.py
index 187aa63cf3..a742047404 100644
--- a/flaml/tune/searcher/flow2.py
+++ b/flaml/tune/searcher/flow2.py
@@ -135,7 +135,6 @@ def __init__(
         self.max_resource = max_resource
         self._resource = None
         self.f_best = None  # only use for lexico_comapre. It represent the best value achieved by lexico_flow.
-        self.op_dimension = None
         self._step_lb = np.Inf
         self.histories = None  # only use for lexico_comapre. It records the result of historical configurations.
         if space is not None:
@@ -396,7 +395,6 @@ def lexico_compare(self, result) -> bool:
                 ):
                     continue
                 elif result[k_metric] < self.best_obj[k_metric]:
-                    self.op_dimension = k_metric
                     return True
                 else:
                     return False
@@ -404,7 +402,6 @@ def lexico_compare(self, result) -> bool:
                 if result[k_metr] == self.best_obj[k_metr]:
                     continue
                 elif result[k_metr] < self.best_obj[k_metr]:
-                    self.op_dimension = k_metric
                     return True
                 else:
                     return False
diff --git a/flaml/tune/searcher/search_thread.py b/flaml/tune/searcher/search_thread.py
index 9bc7e5c705..dd2408b05b 100644
--- a/flaml/tune/searcher/search_thread.py
+++ b/flaml/tune/searcher/search_thread.py
@@ -59,6 +59,7 @@ def __init__(
 
         if self.lexico_objectives:
             # lexicographic tuning setting
+            self.f_best, self.histories = {}, defaultdict(list)  # only use for lexico_comapre.
             self.obj_best1 = self.obj_best2 = {}
             for k_metric in self.lexico_objectives["metrics"]:
                 self.obj_best1[k_metric] = self.obj_best2[k_metric] = (
@@ -71,8 +72,6 @@ def __init__(
             # normal tuning setting
             self.obj_best1 = self.obj_best2 = getattr(search_alg, "best_obj", np.inf)  # inherently minimize
             self.priority = self.speed = 0
-        if self.lexico_objectives:
-            self.f_best, self.histories = {}, defaultdict(list)  # only use for lexico_comapre.
 
     def suggest(self, trial_id: str) -> Optional[Dict]:
         """Use the suggest() of the underlying search algorithm."""
@@ -93,7 +92,8 @@ def suggest(self, trial_id: str) -> Optional[Dict]:
             self.running += 1
         return config
 
-    def update_lexicoinfo(self, result):
+    def update_lexicoPara(self, result):
+        # update histories, f_best
         if self.lexico_objectives:
             for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
                 self.histories[k_metric].append(result[k_metric]) if k_mode == "min" else self.histories[
@@ -217,7 +217,7 @@ def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error:
                 try:
                     self._search_alg.on_trial_complete(trial_id, result, error)
                     if not self._is_ls:
-                        self.update_lexicoinfo(result)
+                        self.update_lexicoPara(result)
                 except RuntimeError as e:
                     # rs is used in place of optuna sometimes
                     if not str(e).endswith("has already finished and can not be updated."):
@@ -267,7 +267,7 @@ def on_trial_result(self, trial_id: str, result: Dict):
             try:
                 self._search_alg.on_trial_result(trial_id, result)
                 if not self._is_ls:
-                    self.update_lexicoinfo(result)
+                    self.update_lexicoPara(result)
             except RuntimeError as e:
                 # rs is used in place of optuna sometimes
                 if not str(e).endswith("has already finished and can not be updated."):