Coverage for src/sparkle/configurator/implementations/smac3.py: 86%
160 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-08 12:00 +0000
1"""Configurator classes to implement SMAC3 in Sparkle."""
3from __future__ import annotations
4from pathlib import Path
6from smac import version as smac_version
7from smac import Scenario as SmacScenario
8from smac import facade as smacfacades
9from smac.runhistory.enumerations import StatusType as SmacStatusType
10import numpy as np
11import random
12from typing import Optional
14from runrunner import Runner, Run
16from sparkle.configurator.configurator import Configurator, ConfigurationScenario
17from sparkle.solver import Solver
18from sparkle.structures import FeatureDataFrame, PerformanceDataFrame
19from sparkle.instance import InstanceSet, Instance_Set
20from sparkle.types import SparkleObjective, resolve_objective, SolverStatus
23class SMAC3(Configurator):
24 """Class for SMAC3 (Python) configurator."""
26 configurator_path = Path(__file__).parent.resolve() / "SMAC3"
27 configurator_target = configurator_path / "smac3_target_algorithm.py"
29 full_name = "Sequential Model-based Algorithm Configuration"
30 version = smac_version
32 def __init__(self: SMAC3) -> None:
33 """Returns the SMAC3 configurator, Python SMAC V2.3.1."""
34 return super().__init__(multi_objective_support=False)
36 @property
37 def name(self: SMAC3) -> str:
38 """Returns the name of the configurator."""
39 return SMAC3.__name__
41 @staticmethod
42 def scenario_class() -> ConfigurationScenario:
43 """Returns the SMAC3 scenario class."""
44 return SMAC3Scenario
46 @staticmethod
47 def check_requirements(verbose: bool = False) -> bool:
48 """Check that SMAC3 is installed."""
49 return True # Is automatically installed with Sparkle
51 @staticmethod
52 def download_requirements() -> None:
53 """Download SMAC3."""
54 return # Nothing to do
56 def configure(
57 self: SMAC3,
58 scenario: SMAC3Scenario,
59 data_target: PerformanceDataFrame,
60 validate_after: bool = True,
61 sbatch_options: list[str] = [],
62 slurm_prepend: str | list[str] | Path = None,
63 num_parallel_jobs: int = None,
64 base_dir: Path = None,
65 run_on: Runner = Runner.SLURM,
66 ) -> list[Run]:
67 """Start configuration job.
69 Args:
70 scenario: ConfigurationScenario object
71 data_target: PerformanceDataFrame where to store the found configurations
72 validate_after: Whether the Validator will be called after the configuration
73 sbatch_options: List of slurm batch options to use
74 slurm_prepend: Slurm script to prepend to the sbatch
75 num_parallel_jobs: The maximum number of jobs to run parallel.
76 base_dir: The path where the sbatch scripts will be created for Slurm.
77 run_on: On which platform to run the jobs. Default: Slurm.
79 Returns:
80 A RunRunner Run object.
81 """
82 scenario.create_scenario()
83 if (
84 scenario.smac3_scenario.walltime_limit
85 == scenario.smac3_scenario.cputime_limit
86 == np.inf
87 ):
88 print("WARNING: Starting SMAC3 scenario without any time limit.")
89 configuration_ids = scenario.configuration_ids
91 # Scenario file also has a seed, but not for all type of configurators
92 seeds = [random.randint(0, 2**32 - 1) for _ in range(scenario.number_of_runs)]
93 num_parallel_jobs = num_parallel_jobs or scenario.number_of_runs
94 # We do not require the configurator CLI as its already our own python wrapper
95 cmds = [
96 f"python3 {self.configurator_target.absolute()} "
97 f"{scenario.scenario_file_path.absolute()} {configuration_id} {seed} "
98 f"{data_target.csv_filepath}"
99 for configuration_id, seed in zip(configuration_ids, seeds)
100 ]
101 return super().configure(
102 configuration_commands=cmds,
103 data_target=data_target,
104 output=None,
105 scenario=scenario,
106 configuration_ids=configuration_ids,
107 validate_after=validate_after,
108 sbatch_options=sbatch_options,
109 slurm_prepend=slurm_prepend,
110 num_parallel_jobs=num_parallel_jobs,
111 base_dir=base_dir,
112 run_on=run_on,
113 )
115 @staticmethod
116 def organise_output(
117 output_source: Path,
118 output_target: Path,
119 scenario: SMAC3Scenario,
120 configuration_id: str,
121 ) -> None | str:
122 """Method to restructure and clean up after a single configurator call."""
123 import json
125 if not output_source.exists():
126 print(f"SMAC3 ERROR: Output source file does not exist! [{output_source}]")
127 return
128 results_dict = json.load(output_source.open("r"))
129 configurations = [value for _, value in results_dict["configs"].items()]
130 config_evals = [[] for _ in range(len(configurations))]
131 objective = scenario.sparkle_objective
132 for entry in results_dict["data"]:
133 smac_conf_id = entry["config_id"]
134 score = entry["cost"]
135 # SMAC3 configuration ids start at 1
136 config_evals[smac_conf_id - 1].append(score)
137 config_evals = [
138 objective.instance_aggregator(evaluations) for evaluations in config_evals
139 ]
140 best_config = configurations[
141 config_evals.index(objective.solver_aggregator(config_evals))
142 ]
143 best_config["configuration_id"] = configuration_id
144 return Configurator.save_configuration(
145 scenario, configuration_id, best_config, output_target
146 )
148 def get_status_from_logs(self: SMAC3) -> None:
149 """Method to scan the log files of the configurator for warnings."""
150 raise NotImplementedError
152 @staticmethod
153 def convert_status(status: SolverStatus) -> SmacStatusType:
154 """Converts Sparkle Solver status to SMAC3 target status."""
155 mapping = {
156 SolverStatus.SUCCESS: SmacStatusType.SUCCESS,
157 SolverStatus.CRASHED: SmacStatusType.CRASHED,
158 SolverStatus.TIMEOUT: SmacStatusType.TIMEOUT,
159 SolverStatus.WRONG: SmacStatusType.CRASHED,
160 SolverStatus.UNKNOWN: SmacStatusType.CRASHED,
161 SolverStatus.ERROR: SmacStatusType.CRASHED,
162 SolverStatus.KILLED: SmacStatusType.TIMEOUT,
163 SolverStatus.SAT: SmacStatusType.SUCCESS,
164 SolverStatus.UNSAT: SmacStatusType.SUCCESS,
165 }
166 return mapping[status]
169class SMAC3Scenario(ConfigurationScenario):
170 """Class to handle SMAC3 configuration scenarios."""
172 def __init__(
173 self: SMAC3Scenario,
174 solver: Solver,
175 instance_set: InstanceSet,
176 sparkle_objectives: list[SparkleObjective],
177 number_of_runs: int,
178 parent_directory: Path,
179 solver_cutoff_time: int = None,
180 smac_facade: smacfacades.AbstractFacade
181 | str = smacfacades.AlgorithmConfigurationFacade,
182 crash_cost: float | list[float] = np.inf,
183 termination_cost_threshold: float | list[float] = np.inf,
184 walltime_limit: float = np.inf,
185 cputime_limit: float = np.inf,
186 solver_calls: int = None,
187 use_default_config: bool = False,
188 feature_data: FeatureDataFrame | Path = None,
189 min_budget: float | int | None = None,
190 max_budget: float | int | None = None,
191 seed: int = -1,
192 n_workers: int = 1,
193 max_ratio: float = None,
194 smac3_output_directory: Path = Path(),
195 timestamp: str = None,
196 ) -> None:
197 """Initialize scenario paths and names.
199 Args:
200 solver: Solver
201 The solver to use for configuration.
202 instance_set: InstanceSet
203 The instance set to use for configuration.
204 sparkle_objectives: list[SparkleObjective]
205 The objectives to optimize.
206 number_of_runs: int
207 The number of times this scenario will be executed with different seeds.
208 parent_directory: Path
209 The parent directory where the configuration files will be stored.
210 solver_cutoff_time: int
211 Maximum CPU runtime in seconds that each solver call (trial)
212 is allowed to run. Is managed by RunSolver, not pynisher.
213 smac_facade: AbstractFacade, defaults to AlgorithmConfigurationFacade
214 The SMAC facade to use for Optimisation.
215 crash_cost: float | list[float], defaults to np.inf
216 Defines the cost for a failed trial. In case of multi-objective,
217 each objective can be associated with a different cost.
218 termination_cost_threshold: float | list[float], defaults to np.inf
219 Defines a cost threshold when the optimization should stop. In case of
220 multi-objective, each objective *must* be associated with a cost.
221 The optimization stops when all objectives crossed the threshold.
222 walltime_limit: float, defaults to np.inf
223 The maximum time in seconds that SMAC is allowed to run. Only counts
224 solver time.
225 cputime_limit: float, defaults to np.inf
226 The maximum CPU time in seconds that SMAC is allowed to run. Only counts
227 solver time.
228 solver_calls: int, defaults to None
229 The maximum number of trials (combination of configuration, seed, budget,
230 and instance, depending on the task) to run. If left as None, will be
231 calculated as int(cutoff time / cputime or walltime limit)
232 use_default_config: bool, defaults to False
233 If True, the configspace's default configuration is evaluated in the
234 initial design. For historic benchmark reasons, this is False by default.
235 Notice, that this will result in n_configs + 1 for the initial design.
236 Respecting n_trials, this will result in one fewer evaluated
237 configuration in the optimization.
238 instances: list[str] | None, defaults to None
239 Names of the instances to use. If None, no instances are used. Instances
240 could be dataset names, seeds, subsets, etc.
241 feature_data: FeatureDataFrame or Path, defaults to None
242 Instances can be associated with features. For example, meta data of
243 the dataset (mean, var, ...) can be incorporated which are then further
244 used to expand the training data of the surrogate model. If Path, loaded
245 from file. When no features are given, uses index as instance features.
246 min_budget: float | int | None, defaults to None
247 The minimum budget (epochs, subset size, number of instances, ...) that
248 is used for the optimization. Use this argument if you use multi-fidelity
249 or instance optimization.
250 max_budget: float | int | None, defaults to None
251 The maximum budget (epochs, subset size, number of instances, ...) that
252 is used for the optimization. Use this argument if you use multi-fidelity
253 or instance optimization.
254 seed: int, defaults to -1
255 The seed is used to make results reproducible.
256 If seed is -1, SMAC will generate a random seed.
257 n_workers: int, defaults to 1
258 The number of workers to use for parallelization.
259 If `n_workers` is greather than 1, SMAC will use DASK to parallelize the
260 optimization.
261 max_ratio: float, defaults to None.
262 Facade uses at most scenario.n_trials * max_ratio number of
263 configurations in the initial design. Additional configurations are not
264 affected by this parameter. Not applicable to each facade.
265 smac3_output_directory: Path, defaults to Path()
266 The output subdirectory for the SMAC3 scenario. Defaults to the scenario
267 results directory.
268 timestamp: An optional timestamp for the directory name.
269 """
270 super().__init__(
271 solver,
272 instance_set,
273 sparkle_objectives,
274 number_of_runs,
275 parent_directory,
276 timestamp,
277 )
278 self.feature_data = feature_data
279 if isinstance(self.feature_data, Path): # Load from file
280 self.feature_data = FeatureDataFrame(self.feature_data)
282 # Facade parameters
283 self.smac_facade = smac_facade
284 if isinstance(self.smac_facade, str):
285 self.smac_facade = getattr(smacfacades, self.smac_facade)
286 self.max_ratio = max_ratio
288 if self.feature_data is not None:
289 instance_features = {
290 path: self.feature_data.get_instance(set_name, instance_name)
291 for path, (set_name, instance_name) in zip(
292 self.instance_set.instance_paths, self.instance_set.instance_pairs
293 )
294 }
295 else:
296 # 'If no instance features are passed, the runhistory encoder can not
297 # distinguish between different instances and therefore returns the same data
298 # points with different values, all of which are used to train the surrogate
299 # model. Consider using instance indices as features.'
300 instance_features = {
301 name: [index] for index, name in enumerate(instance_set.instance_paths)
302 }
304 # NOTE: Patchfix; SMAC3 can handle MO but Sparkle also gives non-user specified
305 # objectives but not all class methods can handle it here yet
306 self.sparkle_objective = sparkle_objectives[0]
308 # NOTE: We don't use trial_walltime_limit as a way of managing resources
309 # As it uses pynisher to do it (python based) and our targets are maybe not
310 # RunSolver is the better option for accuracy.
311 self.solver_cutoff_time = solver_cutoff_time
312 if solver_calls is None: # If solver calls is None, try to calculate it
313 if self.solver_cutoff_time is not None and (cputime_limit or walltime_limit):
314 if cputime_limit:
315 solver_calls = int(cputime_limit / self.solver_cutoff_time)
316 elif walltime_limit:
317 solver_calls = int(walltime_limit / self.solver_cutoff_time)
318 else:
319 solver_calls = 100 # SMAC3 Default value
320 self.smac3_output_directory = smac3_output_directory
321 self.crash_cost = crash_cost
322 self.termination_cost_threshold = termination_cost_threshold
323 self.walltime_limit = walltime_limit
324 self.cputime_limit = cputime_limit
325 self.solver_calls = solver_calls
326 self.use_default_config = use_default_config
327 self.instance_features = instance_features
328 self.min_budget = min_budget
329 self.max_budget = max_budget
330 self.seed = seed
331 self.n_workers = n_workers
332 self.smac3_scenario: Optional[SmacScenario] = None
334 def create_scenario(self: SMAC3Scenario) -> None:
335 """This prepares all the necessary subdirectories related to configuration."""
336 super().create_scenario()
337 self.log_dir.mkdir(parents=True)
338 if self.smac3_scenario is None:
339 self.set_smac3_scenario()
340 self.create_scenario_file()
342 def set_smac3_scenario(self: SMAC3Scenario) -> None:
343 """Set the smac scenario object."""
344 self.smac3_scenario = SmacScenario(
345 configspace=self.solver.get_configuration_space(),
346 name=self.name,
347 output_directory=self.results_directory / self.smac3_output_directory,
348 deterministic=self.solver.deterministic,
349 objectives=[self.sparkle_objective.name],
350 crash_cost=self.crash_cost,
351 termination_cost_threshold=self.termination_cost_threshold,
352 walltime_limit=self.walltime_limit,
353 cputime_limit=self.cputime_limit,
354 n_trials=self.solver_calls,
355 use_default_config=self.use_default_config,
356 instances=self.instance_set.instance_paths,
357 instance_features=self.instance_features,
358 min_budget=self.min_budget,
359 max_budget=self.max_budget,
360 seed=self.seed,
361 n_workers=self.n_workers,
362 )
364 @property
365 def log_dir(self: SMAC3Scenario) -> Path:
366 """Return the path of the log directory."""
367 if self.directory:
368 return self.directory / "logs"
369 return None
371 @property
372 def configurator(self: SMAC3Scenario) -> SMAC3:
373 """Return the type of configurator the scenario belongs to."""
374 return SMAC3
376 def create_scenario_file(self: SMAC3Scenario) -> Path:
377 """Create a file with the configuration scenario."""
378 with self.scenario_file_path.open("w") as file:
379 for key, value in self.serialise().items():
380 file.write(f"{key} = {value}\n")
382 def serialise(self: SMAC3Scenario) -> dict:
383 """Serialize the configuration scenario."""
384 feature_data = str(self.feature_data.csv_filepath) if self.feature_data else None
385 return {
386 "solver": self.solver.directory,
387 "instance_set": self.instance_set.directory,
388 "sparkle_objectives": ",".join(self.smac3_scenario.objectives),
389 "solver_cutoff_time": self.solver_cutoff_time,
390 "number_of_runs": self.number_of_runs,
391 "smac_facade": self.smac_facade.__name__,
392 "crash_cost": self.smac3_scenario.crash_cost,
393 "termination_cost_threshold": self.smac3_scenario.termination_cost_threshold,
394 "walltime_limit": self.smac3_scenario.walltime_limit,
395 "cputime_limit": self.smac3_scenario.cputime_limit,
396 "solver_calls": self.smac3_scenario.n_trials,
397 "use_default_config": self.smac3_scenario.use_default_config,
398 "feature_data": feature_data,
399 "min_budget": self.smac3_scenario.min_budget,
400 "max_budget": self.smac3_scenario.max_budget,
401 "seed": self.smac3_scenario.seed,
402 "n_workers": self.smac3_scenario.n_workers,
403 }
405 @staticmethod
406 def from_file(scenario_file: Path, run_index: int = None) -> SMAC3Scenario:
407 """Reads scenario file and initalises ConfigurationScenario.
409 Args:
410 scenario_file: Path to scenario file.
411 run_index: If given, reads as the scenario with run_index for offset
412 in output directory and seed.
414 Returns:
415 ConfigurationScenario.
416 """
417 import ast
419 variables = {
420 keyvalue[0]: keyvalue[1].strip()
421 for keyvalue in (
422 line.split(" = ", maxsplit=1)
423 for line in scenario_file.open().readlines()
424 if line.strip() != ""
425 )
426 }
427 variables["solver"] = Solver(Path(variables["solver"]))
428 variables["instance_set"] = Instance_Set(Path(variables["instance_set"]))
429 variables["sparkle_objectives"] = [
430 resolve_objective(o) for o in variables["sparkle_objectives"].split(",")
431 ]
432 variables["parent_directory"] = scenario_file.parent.parent
433 variables["solver_cutoff_time"] = int(variables["solver_cutoff_time"])
434 variables["number_of_runs"] = int(variables["number_of_runs"])
435 variables["smac_facade"] = getattr(smacfacades, variables["smac_facade"])
437 # We need to support both lists of floats and single float (np.inf is fine)
438 if variables["crash_cost"].startswith("["):
439 variables["crash_cost"] = [
440 float(v) for v in ast.literal_eval(variables["crash_cost"])
441 ]
442 else:
443 variables["crash_cost"] = float(variables["crash_cost"])
444 if variables["termination_cost_threshold"].startswith("["):
445 variables["termination_cost_threshold"] = [
446 float(v)
447 for v in ast.literal_eval(variables["termination_cost_threshold"])
448 ]
449 else:
450 variables["termination_cost_threshold"] = float(
451 variables["termination_cost_threshold"]
452 )
454 variables["walltime_limit"] = float(variables["walltime_limit"])
455 variables["cputime_limit"] = float(variables["cputime_limit"])
456 variables["solver_calls"] = ast.literal_eval(variables["solver_calls"])
457 variables["use_default_config"] = ast.literal_eval(
458 variables["use_default_config"]
459 )
461 if variables["feature_data"] != "None":
462 variables["feature_data"] = Path(variables["feature_data"])
463 else:
464 variables["feature_data"] = None
466 variables["min_budget"] = ast.literal_eval(variables["min_budget"])
467 variables["max_budget"] = ast.literal_eval(variables["max_budget"])
469 variables["seed"] = ast.literal_eval(variables["seed"])
470 variables["n_workers"] = ast.literal_eval(variables["n_workers"])
471 if run_index is not None: # Offset
472 variables["seed"] += run_index
473 variables["smac3_output_directory"] = Path(f"run_{run_index}")
475 timestamp = scenario_file.parent.name.split("_")[-1]
476 scenario = SMAC3Scenario(**variables, timestamp=timestamp)
477 scenario.set_smac3_scenario()
478 return scenario