Coverage for src/sparkle/configurator/implementations/smac2.py: 75%
204 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-08 12:00 +0000
1"""Configurator classes to implement SMAC2 in Sparkle."""
3from __future__ import annotations
4from pathlib import Path
5import shutil
6import math
7import random
9import pandas as pd
11from runrunner import Runner, Run
13from sparkle.tools.parameters import PCSConvention
14from sparkle.configurator.configurator import Configurator, ConfigurationScenario
15from sparkle.solver import Solver
16from sparkle.structures import PerformanceDataFrame, FeatureDataFrame
17from sparkle.instance import InstanceSet, Instance_Set
18from sparkle.types import SparkleObjective, resolve_objective
21class SMAC2(Configurator):
22 """Class for SMAC2 (Java) configurator."""
24 configurator_path = Path(__file__).parent.resolve() / "SMAC2"
25 configurator_executable = configurator_path / "smac"
26 configurator_target = configurator_path / "smac2_target_algorithm.py"
28 full_name = "Sequential Model-based Algorithm Configuration"
29 version = "2.10.03"
31 def __init__(self: SMAC2) -> None:
32 """Returns the SMAC2 configurator, Java SMAC V2.10.03."""
33 return super().__init__(multi_objective_support=False)
35 @property
36 def name(self: SMAC2) -> str:
37 """Returns the name of the configurator."""
38 return SMAC2.__name__
40 @staticmethod
41 def scenario_class() -> ConfigurationScenario:
42 """Returns the SMAC2 scenario class."""
43 return SMAC2Scenario
45 @staticmethod
46 def check_requirements(verbose: bool = False) -> bool:
47 """Check that SMAC2 is installed."""
48 import warnings
50 if no_java := shutil.which("java") is None:
51 if verbose:
52 warnings.warn(
53 "SMAC2 requires Java 1.8.0_402, but Java is not installed. "
54 "Please ensure Java is installed."
55 )
56 if no_smac := not SMAC2.configurator_executable.exists():
57 if verbose:
58 warnings.warn(
59 "SMAC2 executable not found. Please ensure SMAC2 is installed "
60 f"in the expected Path ({SMAC2.configurator_path})."
61 )
62 return not (no_java or no_smac)
64 @staticmethod
65 def download_requirements(
66 smac2_zip_url: str = "https://github.com/ADA-research/Sparkle/raw/refs/heads/"
67 "development/Resources/Configurators/SMAC2-v2.10.03.zip",
68 ) -> None:
69 """Download SMAC2."""
70 if SMAC2.configurator_executable.exists():
71 return # Already installed
72 from urllib.request import urlopen
73 import zipfile
74 import io
76 r = urlopen(smac2_zip_url, timeout=60)
77 z = zipfile.ZipFile(io.BytesIO(r.read()))
78 z.extractall(SMAC2.configurator_path)
79 # Ensure execution rights
80 SMAC2.configurator_executable.chmod(0o755)
82 def configure(
83 self: SMAC2,
84 scenario: SMAC2Scenario,
85 data_target: PerformanceDataFrame,
86 validate_after: bool = True,
87 sbatch_options: list[str] = [],
88 slurm_prepend: str | list[str] | Path = None,
89 num_parallel_jobs: int = None,
90 base_dir: Path = None,
91 run_on: Runner = Runner.SLURM,
92 ) -> list[Run]:
93 """Start configuration job.
95 Args:
96 scenario: ConfigurationScenario object
97 data_target: PerformanceDataFrame where to store the found configurations
98 validate_after: Whether the configurations should be validated on the
99 train set afterwards.
100 sbatch_options: List of slurm batch options to use.
101 slurm_prepend: Slurm script to prepend to the sbatch.
102 num_parallel_jobs: The maximum number of jobs to run parallel.
103 base_dir: The path where the sbatch scripts will be created for Slurm.
104 run_on: On which platform to run the jobs. Default: Slurm.
106 Returns:
107 A RunRunner Run object.
108 """
109 scenario.create_scenario()
110 configuration_ids = scenario.configuration_ids
112 # The maximum seed size for SMAC2 is 999 999 999
113 seeds = [random.randint(0, 10**9 - 1) for _ in range(scenario.number_of_runs)]
114 output = [
115 f"{(scenario.results_directory).absolute()}/"
116 f"{scenario.name}_{config_id}_smac.txt"
117 for config_id in configuration_ids
118 ]
119 cmds = [
120 f"python3 {Configurator.configurator_cli_path.absolute()} "
121 f"{SMAC2.__name__} {output_file} {data_target.csv_filepath} "
122 f"{scenario.scenario_file_path} {configuration_id} "
123 f"{SMAC2.configurator_executable.absolute()} "
124 f"--scenario-file {scenario.scenario_file_path} "
125 f"--seed {seed} "
126 for output_file, configuration_id, seed in zip(
127 output, configuration_ids, seeds
128 )
129 ]
130 if num_parallel_jobs is not None:
131 num_parallel_jobs = max(num_parallel_jobs, len(cmds))
132 return super().configure(
133 configuration_commands=cmds,
134 data_target=data_target,
135 output=output,
136 num_parallel_jobs=num_parallel_jobs,
137 scenario=scenario,
138 configuration_ids=configuration_ids,
139 validate_after=validate_after,
140 sbatch_options=sbatch_options,
141 slurm_prepend=slurm_prepend,
142 base_dir=base_dir,
143 run_on=run_on,
144 )
146 @staticmethod
147 def organise_output(
148 output_source: Path,
149 output_target: Path,
150 scenario: SMAC2Scenario,
151 configuration_id: str,
152 ) -> None | dict:
153 """Retrieves configuration from SMAC file and places them in output."""
154 call_key = SMAC2.configurator_target.name
155 # Last line describing a call is the best found configuration
156 for line in reversed(output_source.open("r").readlines()):
157 if call_key in line:
158 call_str = line.split(call_key, maxsplit=1)[1].strip()
159 # The Configuration appears after the first 7 arguments
160 configuration = call_str.split(" ", 8)[-1]
161 break
162 configuration = Solver.config_str_to_dict(configuration)
163 configuration["configuration_id"] = configuration_id
164 return Configurator.save_configuration(
165 scenario, configuration_id, configuration, output_target
166 )
168 @staticmethod
169 def get_smac_run_obj(objective: SparkleObjective) -> str:
170 """Return the SMAC run objective based on the Performance Measure.
172 Returns:
173 A string that represents the run objective set in the settings.
174 """
175 if objective.time:
176 return "RUNTIME"
177 return "QUALITY"
179 def get_status_from_logs(self: SMAC2, base_dir: Path) -> None:
180 """Method to scan the log files of the configurator for warnings."""
181 if not base_dir.exists():
182 return
183 print(
184 f"Checking the log files of configurator {type(self).__name__} for "
185 "warnings..."
186 )
187 scenarios = [f for f in base_dir.iterdir() if f.is_dir()]
188 for scenario in scenarios:
189 log_dir = (
190 scenario / "outdir_train_configuration" / (scenario.name + "_scenario")
191 )
192 # Collect all non empty log files paths
193 warn_files = [f for f in log_dir.glob("log-warn*") if f.stat().st_size > 0]
194 if len(warn_files) > 0:
195 print(
196 f"Scenario {scenario.name} has {len(warn_files)} warning(s), see "
197 "the following log file(s) for more information:"
198 )
199 for log_file in warn_files:
200 print(f"\t-{log_file}")
201 else:
202 print(f"Scenario {scenario.name} has no warnings.")
205class SMAC2Scenario(ConfigurationScenario):
206 """Class to handle SMAC2 configuration scenarios."""
208 def __init__(
209 self: SMAC2Scenario,
210 solver: Solver,
211 instance_set: InstanceSet,
212 sparkle_objectives: list[SparkleObjective],
213 number_of_runs: int,
214 parent_directory: Path,
215 solver_calls: int = None,
216 max_iterations: int = None,
217 cpu_time: int = None,
218 wallclock_time: int = None,
219 solver_cutoff_time: int = None,
220 target_cutoff_length: str = None,
221 cli_cores: int = None,
222 use_cpu_time_in_tunertime: bool = None,
223 feature_data: FeatureDataFrame | Path = None,
224 timestamp: str = None,
225 ) -> None:
226 """Initialize scenario paths and names.
228 Args:
229 solver: Solver that should be configured.
230 instance_set: Instances object for the scenario.
231 sparkle_objectives: SparkleObjectives used for each run of the configuration.
232 Will be simplified to the first objective.
233 number_of_runs: The number of configurator runs to perform
234 for configuring the solver.
235 parent_directory: Directory in which the scenario should be created.
236 solver_calls: The number of times the solver is called for each
237 configuration run
238 max_iterations: The maximum number of iterations allowed for each
239 configuration run. [iteration-limit, numIterations, numberOfIterations]
240 cpu_time: The time budget allocated for each configuration run. (cpu)
241 wallclock_time: The time budget allocated for each configuration run.
242 (wallclock)
243 solver_cutoff_time: The maximum time allowed for each solver call run during
244 configuration.
245 target_cutoff_length: A domain specific measure of when the algorithm
246 should consider itself done.
247 cli_cores: int
248 The number of cores to use to execute runs. Defaults in SMAC2 to 1.
249 use_cpu_time_in_tunertime: Whether to calculate SMAC2's own used time for
250 budget deduction. Defaults in SMAC2 to True.
251 feature_data: If features are used, this contains the feature data.
252 If it is a FeatureDataFrame, will convert values to SMAC2 format.
253 If it is a Path, will pass the path to SMAC2.
254 Defaults to None.
255 timestamp: An optional timestamp for the directory name.
256 """
257 super().__init__(
258 solver,
259 instance_set,
260 sparkle_objectives,
261 number_of_runs,
262 parent_directory,
263 timestamp,
264 )
265 self.solver = solver
266 self.instance_set = instance_set
268 self.sparkle_objective = sparkle_objectives[0]
269 self.solver_calls = solver_calls
270 self.cpu_time = cpu_time
271 self.wallclock_time = wallclock_time
272 self.solver_cutoff_time = solver_cutoff_time
273 self.cutoff_length = target_cutoff_length
274 self.max_iterations = max_iterations
275 self.cli_cores = cli_cores
276 self.use_cpu_time_in_tunertime = use_cpu_time_in_tunertime
278 self.feature_data = feature_data
279 self._feature_file_path = None
280 if self.feature_data:
281 if isinstance(self.feature_data, FeatureDataFrame):
282 # Convert feature data to SMAC2 format
283 data_dict = {}
284 for path, (set_name, instance_name) in zip(
285 self.instance_set.instance_paths, self.instance_set.instance_pairs
286 ):
287 data_dict[str(path)] = feature_data.get_instance(
288 set_name, instance_name
289 )
291 self.feature_data = pd.DataFrame.from_dict(
292 data_dict,
293 orient="index",
294 columns=[
295 f"Feature{index + 1}"
296 for index in range(feature_data.num_features)
297 ],
298 )
300 def map_nan(x: str) -> int:
301 """Map non-numeric values with -512 (Pre-defined by SMAC2)."""
302 if math.isnan(x):
303 return -512.0
304 try:
305 return float(x)
306 except Exception:
307 return -512.0
309 self.feature_data = self.feature_data.map(map_nan)
310 elif isinstance(self.feature_data, Path): # Read from Path
311 self._feature_file_path = feature_data
312 self.feature_data = pd.read_csv(self.feature_file_path, index_col=0)
313 else:
314 print(
315 f"WARNING: Feature data is of type {type(feature_data)}. "
316 "Expected FeatureDataFrame or Path."
317 )
319 @property
320 def instance_file_path(self: SMAC2Scenario) -> Path:
321 """Return the path of the instance file."""
322 if self.directory:
323 return self.directory / f"{self.instance_set.name}.txt"
324 return None
326 @property
327 def outdir_train(self: SMAC2Scenario) -> Path:
328 """Return the path of the train out directory."""
329 # SMAC2 Specific directory
330 if self.directory:
331 return self.directory / "outdir_train_configuration"
332 return None
334 @property
335 def feature_file_path(self: SMAC2Scenario) -> Path:
336 """Return the path of the feature file."""
337 if self._feature_file_path:
338 return self._feature_file_path
339 elif self.directory:
340 return self.directory / f"{self.instance_set.name}_features.csv"
341 else:
342 return None
344 @property
345 def configurator(self: SMAC2Scenario) -> SMAC2:
346 """Return the type of configurator the scenario belongs to."""
347 return SMAC2
349 def create_scenario(self: SMAC2Scenario) -> None:
350 """Create scenario with solver and instances in the parent directory.
352 This prepares all the necessary subdirectories related to configuration.
354 Args:
355 parent_directory: Directory in which the scenario should be created.
356 """
357 super().create_scenario()
358 self.outdir_train.mkdir()
359 self._prepare_instances()
361 if self.feature_data is not None:
362 self._create_feature_file()
364 self.create_scenario_file()
366 def create_scenario_file(
367 self: SMAC2Scenario,
368 configurator_target: Path = SMAC2.configurator_target,
369 pcs_port: PCSConvention = PCSConvention.SMAC,
370 ) -> Path:
371 """Create a file with the configuration scenario.
373 Writes supplementary information to the target algorithm (algo =) as:
374 algo = {configurator_target} {solver_directory} {sparkle_objective}
375 """
376 with self.scenario_file_path.open("w") as file:
377 file.write(
378 f"algo = {configurator_target.absolute()} "
379 f"{self.solver.directory} {self.tmp} {self.sparkle_objective} \n"
380 f"deterministic = {1 if self.solver.deterministic else 0}\n"
381 f"run_obj = {self._get_performance_measure()}\n"
382 f"cutoffTime = {self.solver_cutoff_time}\n"
383 f"paramfile = {self.solver.get_pcs_file(pcs_port)}\n"
384 f"outdir = {self.outdir_train}\n"
385 f"instance_file = {self.instance_file_path}\n"
386 f"test_instance_file = {self.instance_file_path}\n"
387 )
388 if self.cutoff_length is not None:
389 file.write(f"cutoff_length = {self.cutoff_length}\n")
390 if self.max_iterations is not None:
391 file.write(f"iteration-limit = {self.max_iterations}\n")
392 if self.wallclock_time is not None:
393 file.write(f"wallclock-limit = {self.wallclock_time}\n")
394 if self.cpu_time is not None:
395 file.write(f"cputime-limit = {self.cpu_time}\n")
396 if self.solver_calls is not None:
397 file.write(f"runcount-limit = {self.solver_calls}\n")
398 if self.cli_cores is not None:
399 file.write(f"cli-cores = {self.cli_cores}")
400 if self.feature_data is not None:
401 file.write(f"feature_file = {self.feature_file_path}\n")
402 if self.use_cpu_time_in_tunertime is not None:
403 file.write(
404 f"use-cpu-time-in-tunertime = {self.use_cpu_time_in_tunertime}\n"
405 )
406 # We don't let SMAC do the validation
407 file.write("validation = false" + "\n")
408 return self.scenario_file_path
410 def _prepare_instances(self: SMAC2Scenario) -> None:
411 """Create instance list file without instance specifics."""
412 self.instance_file_path.parent.mkdir(exist_ok=True, parents=True)
413 with self.instance_file_path.open("w+") as file:
414 for instance_path in self.instance_set._instance_paths:
415 file.write(f"{instance_path}\n")
417 def _create_feature_file(self: SMAC2Scenario) -> None:
418 """Create CSV file from feature data."""
419 self.feature_data.to_csv(self.feature_file_path, index_label="INSTANCE_NAME")
421 def _get_performance_measure(self: SMAC2Scenario) -> str:
422 """Retrieve the performance measure of the SparkleObjective.
424 Returns:
425 Performance measure of the sparkle objective
426 """
427 if self.sparkle_objective.time:
428 return "RUNTIME"
429 return "QUALITY"
431 def serialise(self: SMAC2Scenario) -> dict:
432 """Transform ConfigurationScenario to dictionary format."""
433 return {
434 "number_of_runs": self.number_of_runs,
435 "solver_calls": self.solver_calls,
436 "cpu_time": self.cpu_time,
437 "wallclock_time": self.wallclock_time,
438 "solver_cutoff_time": self.solver_cutoff_time,
439 "cutoff_length": self.cutoff_length,
440 "max_iterations": self.max_iterations,
441 "sparkle_objective": self.sparkle_objective.name,
442 "feature_data": str(self.feature_file_path),
443 "use_cpu_time_in_tunertime": self.use_cpu_time_in_tunertime,
444 }
446 @staticmethod
447 def from_file(scenario_file: Path) -> SMAC2Scenario:
448 """Reads scenario file and initalises SMAC2Scenario."""
449 config = {
450 keyvalue[0]: keyvalue[1]
451 for keyvalue in (
452 line.strip().split(" = ", maxsplit=1)
453 for line in scenario_file.open().readlines()
454 if line.strip() != ""
455 )
456 }
458 # Collect relevant settings
459 cpu_time = int(config["cpu_time"]) if "cpu_time" in config else None
460 wallclock_limit = (
461 int(config["wallclock-limit"]) if "wallclock-limit" in config else None
462 )
463 solver_calls = (
464 int(config["runcount-limit"]) if "runcount-limit" in config else None
465 )
466 max_iterations = (
467 int(config["iteration-limit"]) if "iteration-limit" in config else None
468 )
469 use_cpu_time_in_tunertime = (
470 config["use-cputime-in-tunertime"]
471 if "use-cputime-in-tunertime" in config
472 else None
473 )
474 cli_cores = config["cli-cores"] if "cli-cores" in config else None
476 _, solver_path, _, objective_str = config["algo"].split(" ")
477 objective = resolve_objective(objective_str)
478 solver = Solver(Path(solver_path.strip()))
479 # Extract the instance set from the instance file
480 instance_file_path = Path(config["instance_file"])
481 instance_set_path = Path(instance_file_path.open().readline().strip()).parent
482 instance_set = Instance_Set(Path(instance_set_path))
483 results_folder = scenario_file.parent / "results"
484 state_run_dirs = [p for p in results_folder.iterdir() if p.is_file()]
485 number_of_runs = len(state_run_dirs)
486 feature_data_path = None
487 if "feature_file" in config:
488 feature_data_path = Path(config["feature_file"])
489 # Get the timestamp from the scenario dir name
490 timestamp = scenario_file.parent.name.split("_")[-1]
491 return SMAC2Scenario(
492 solver,
493 instance_set,
494 [objective],
495 number_of_runs,
496 instance_file_path.parent.parent,
497 solver_calls,
498 max_iterations,
499 cpu_time,
500 wallclock_limit,
501 int(config["cutoffTime"]),
502 config["cutoff_length"],
503 cli_cores,
504 use_cpu_time_in_tunertime,
505 feature_data_path,
506 timestamp,
507 )