Coverage for sparkle/platform/settings_objects.py: 81%
929 statements
« prev ^ index » next coverage.py v7.6.10, created at 2025-01-07 15:22 +0000
1"""Classes and Enums to control settings."""
2from __future__ import annotations
3import configparser
4from enum import Enum
5import ast
6from pathlib import Path
7from pathlib import PurePath
9from sparkle.types import SparkleObjective, resolve_objective
10from sparkle.types.objective import PAR
11from sparkle.solver import Selector
12from sparkle.configurator.configurator import Configurator
13from sparkle.configurator import implementations as cim
15from runrunner import Runner
16from sparkle.platform.cli_types import VerbosityLevel
19class SettingState(Enum):
20 """Enum of possible setting states."""
22 NOT_SET = 0
23 DEFAULT = 1
24 FILE = 2
25 CMD_LINE = 3
28class Settings:
29 """Class to read, write, set, and get settings."""
30 # CWD Prefix
31 cwd_prefix = Path() # Empty for now
33 # Library prefix
34 lib_prefix = Path(__file__).parent.parent.resolve()
36 # Default directory names
37 rawdata_dir = Path("Raw_Data")
38 analysis_dir = Path("Analysis")
39 DEFAULT_settings_dir = Path("Settings")
40 __settings_file = Path("sparkle_settings.ini")
42 # Default settings path
43 DEFAULT_settings_path = PurePath(cwd_prefix / DEFAULT_settings_dir / __settings_file)
44 DEFAULT_reference_dir = DEFAULT_settings_dir / "Reference_Lists"
46 # Default library pathing
47 DEFAULT_components = lib_prefix / "Components"
49 # Example settings path
50 DEFAULT_example_settings_path = PurePath(DEFAULT_components / "sparkle_settings.ini")
52 # Runsolver component
53 DEFAULT_runsolver_dir = DEFAULT_components / "runsolver" / "src"
54 DEFAULT_runsolver_exec = DEFAULT_runsolver_dir / "runsolver"
56 # Ablation component
57 DEFAULT_ablation_dir = DEFAULT_components / "ablationAnalysis-0.9.4"
58 DEFAULT_ablation_exec = DEFAULT_ablation_dir / "ablationAnalysis"
59 DEFAULT_ablation_validation_exec = DEFAULT_ablation_dir / "ablationValidation"
61 # Autofolio component
62 DEFAULT_general_sparkle_selector = DEFAULT_components / "AutoFolio/scripts/autofolio"
64 # Report component
65 DEFAULT_latex_source = DEFAULT_components / "Sparkle-latex-source"
66 DEFAULT_latex_bib = DEFAULT_latex_source / "SparkleReport.bib"
68 # Default input directory pathing
69 DEFAULT_solver_dir = cwd_prefix / "Solvers"
70 DEFAULT_instance_dir = cwd_prefix / "Instances"
71 DEFAULT_extractor_dir = cwd_prefix / "Extractors"
72 DEFAULT_snapshot_dir = cwd_prefix / "Snapshots"
74 # Default output directory pathing
75 DEFAULT_tmp_output = cwd_prefix / "Tmp"
76 DEFAULT_output = cwd_prefix / "Output"
77 DEFAULT_configuration_output = DEFAULT_output / "Configuration"
78 DEFAULT_selection_output = DEFAULT_output / "Selection"
79 DEFAULT_parallel_portfolio_output = DEFAULT_output / "Parallel_Portfolio"
80 DEFAULT_ablation_output = DEFAULT_output / "Ablation"
81 DEFAULT_log_output = DEFAULT_output / "Log"
83 # Default output subdirs
84 DEFAULT_configuration_output_raw = DEFAULT_configuration_output / rawdata_dir
85 DEFAULT_configuration_output_analysis = DEFAULT_configuration_output / analysis_dir
86 DEFAULT_selection_output_raw = DEFAULT_selection_output / rawdata_dir
87 DEFAULT_selection_output_analysis = DEFAULT_selection_output / analysis_dir
88 DEFAULT_parallel_portfolio_output_raw =\
89 DEFAULT_parallel_portfolio_output / rawdata_dir
90 DEFAULT_parallel_portfolio_output_analysis =\
91 DEFAULT_parallel_portfolio_output / analysis_dir
93 # Old default output dirs which should be part of something else
94 DEFAULT_feature_data = DEFAULT_output / "Feature_Data"
95 DEFAULT_performance_data = DEFAULT_output / "Performance_Data"
97 # Collection of all working dirs for platform
98 DEFAULT_working_dirs = [
99 DEFAULT_solver_dir, DEFAULT_instance_dir, DEFAULT_extractor_dir,
100 DEFAULT_output, DEFAULT_configuration_output,
101 DEFAULT_selection_output,
102 DEFAULT_tmp_output, DEFAULT_log_output,
103 DEFAULT_feature_data, DEFAULT_performance_data,
104 DEFAULT_settings_dir, DEFAULT_reference_dir,
105 ]
107 # Old default file paths from GV which should be turned into variables
108 DEFAULT_feature_data_path =\
109 DEFAULT_feature_data / "feature_data.csv"
110 DEFAULT_performance_data_path =\
111 DEFAULT_performance_data / "performance_data.csv"
113 # Constant default values
114 DEFAULT_general_sparkle_objective = PAR(10)
115 DEFAULT_general_sparkle_configurator = cim.SMAC2.__name__
116 DEFAULT_general_target_cutoff_time = 60
117 DEFAULT_general_extractor_cutoff_time = 60
118 DEFAULT_number_of_jobs_in_parallel = 25
119 DEFAULT_general_verbosity = VerbosityLevel.STANDARD
120 DEFAULT_general_check_interval = 10
121 DEFAULT_general_run_on = "local"
123 DEFAULT_configurator_number_of_runs = 25
124 DEFAULT_configurator_solver_calls = 100
125 DEFAULT_configurator_maximum_iterations = None
127 # Default SMAC2 settings
128 DEFAULT_smac2_wallclock_time = None
129 DEFAULT_smac2_cpu_time = None
130 DEFAULT_smac2_target_cutoff_length = "max"
131 DEFAULT_smac2_use_cpu_time_in_tunertime = None
132 DEFAULT_smac2_cli_cores = None
133 DEFAULT_smac2_max_iterations = None
135 # Default SMAC3 settings
136 DEFAULT_smac3_number_of_runs = None
137 DEFAULT_smac3_facade = "AlgorithmConfigurationFacade"
138 DEFAULT_smac3_facade_max_ratio = None
139 DEFAULT_smac3_crash_cost = None
140 DEFAULT_smac3_termination_cost_threshold = None
141 DEFAULT_smac3_walltime_limit = None
142 DEFAULT_smac3_cputime_limit = None
143 DEFAULT_smac3_use_default_config = None
144 DEFAULT_smac3_min_budget = None
145 DEFAULT_smac3_max_budget = None
147 # Default IRACE settings
148 DEFAULT_irace_max_time = 0 # IRACE equivalent of None in this case
149 DEFAULT_irace_max_experiments = 0
150 DEFAULT_irace_first_test = None
151 DEFAULT_irace_mu = None
152 DEFAULT_irace_max_iterations = None
154 DEFAULT_portfolio_construction_timeout = None
156 DEFAULT_slurm_max_parallel_runs_per_node = 8
158 DEFAULT_ablation_racing = False
160 DEFAULT_parallel_portfolio_check_interval = 4
161 DEFAULT_parallel_portfolio_num_seeds_per_solver = 1
163 # Default IRACE settings
164 DEFAULT_irace_max_time = 0 # IRACE equivalent of None in this case
165 DEFAULT_irace_max_experiments = 0
166 DEFAULT_irace_first_test = None
167 DEFAULT_irace_mu = None
168 DEFAULT_irace_max_iterations = None
170 def __init__(self: Settings, file_path: PurePath = None) -> None:
171 """Initialise a settings object."""
172 # Settings 'dictionary' in configparser format
173 self.__settings = configparser.ConfigParser()
175 # Setting flags
176 self.__general_sparkle_objective_set = SettingState.NOT_SET
177 self.__general_sparkle_configurator_set = SettingState.NOT_SET
178 self.__general_sparkle_selector_set = SettingState.NOT_SET
179 self.__general_target_cutoff_time_set = SettingState.NOT_SET
180 self.__general_extractor_cutoff_time_set = SettingState.NOT_SET
181 self.__general_verbosity_set = SettingState.NOT_SET
182 self.__general_check_interval_set = SettingState.NOT_SET
184 self.__config_solver_calls_set = SettingState.NOT_SET
185 self.__config_number_of_runs_set = SettingState.NOT_SET
186 self.__config_max_iterations_set = SettingState.NOT_SET
188 self.__smac2_wallclock_time_set = SettingState.NOT_SET
189 self.__smac2_cpu_time_set = SettingState.NOT_SET
190 self.__smac2_use_cpu_time_in_tunertime_set = SettingState.NOT_SET
191 self.__smac2_cli_cores_set = SettingState.NOT_SET
192 self.__smac2_max_iterations_set = SettingState.NOT_SET
193 self.__smac2_target_cutoff_length_set = SettingState.NOT_SET
195 self.__smac3_number_of_trials_set = SettingState.NOT_SET
196 self.__smac3_smac_facade_set = SettingState.NOT_SET
197 self.__smac3_facade_max_ratio_set = SettingState.NOT_SET
198 self.__smac3_crash_cost_set = SettingState.NOT_SET
199 self.__smac3_termination_cost_threshold_set = SettingState.NOT_SET
200 self.__smac3_walltime_limit_set = SettingState.NOT_SET
201 self.__smac3_cputime_limit_set = SettingState.NOT_SET
202 self.__smac3_use_default_config_set = SettingState.NOT_SET
203 self.__smac3_min_budget_set = SettingState.NOT_SET
204 self.__smac3_max_budget_set = SettingState.NOT_SET
206 self.__run_on_set = SettingState.NOT_SET
207 self.__number_of_jobs_in_parallel_set = SettingState.NOT_SET
208 self.__slurm_max_parallel_runs_per_node_set = SettingState.NOT_SET
209 self.__ablation_racing_flag_set = SettingState.NOT_SET
211 self.__parallel_portfolio_check_interval_set = SettingState.NOT_SET
212 self.__parallel_portfolio_num_seeds_per_solver_set = SettingState.NOT_SET
214 self.__irace_max_time_set = SettingState.NOT_SET
215 self.__irace_max_experiments_set = SettingState.NOT_SET
216 self.__irace_first_test_set = SettingState.NOT_SET
217 self.__irace_mu_set = SettingState.NOT_SET
218 self.__irace_max_iterations_set = SettingState.NOT_SET
220 self.__general_sparkle_configurator = None
222 self.__slurm_extra_options_set = dict()
224 if file_path is None:
225 # Initialise settings from default file path
226 self.read_settings_ini()
227 else:
228 # Initialise settings from a given file path
229 self.read_settings_ini(file_path)
231 def read_settings_ini(self: Settings, file_path: PurePath = DEFAULT_settings_path,
232 state: SettingState = SettingState.FILE) -> None:
233 """Read the settings from an INI file."""
234 # Read file
235 file_settings = configparser.ConfigParser()
236 file_settings.read(file_path)
238 # Set internal settings based on data read from FILE if they were read
239 # successfully
240 if file_settings.sections() != []:
241 section = "general"
242 option_names = ("objectives", )
243 for option in option_names:
244 if file_settings.has_option(section, option):
245 value = [resolve_objective(obj) for obj in
246 file_settings.get(section, option).split(",")]
247 self.set_general_sparkle_objectives(value, state)
248 file_settings.remove_option(section, option)
250 # Comma so python understands it's a tuple...
251 option_names = ("configurator", )
252 for option in option_names:
253 if file_settings.has_option(section, option):
254 value = file_settings.get(section, option)
255 self.set_general_sparkle_configurator(value, state)
256 file_settings.remove_option(section, option)
258 option_names = ("selector", )
259 for option in option_names:
260 if file_settings.has_option(section, option):
261 value = file_settings.get(section, option)
262 self.set_general_sparkle_selector(value, state)
263 file_settings.remove_option(section, option)
265 option_names = ("target_cutoff_time",
266 "cutoff_time_each_solver_call")
267 for option in option_names:
268 if file_settings.has_option(section, option):
269 value = file_settings.getint(section, option)
270 self.set_general_target_cutoff_time(value, state)
271 file_settings.remove_option(section, option)
273 option_names = ("extractor_cutoff_time",
274 "cutoff_time_each_feature_computation")
275 for option in option_names:
276 if file_settings.has_option(section, option):
277 value = file_settings.getint(section, option)
278 self.set_general_extractor_cutoff_time(value, state)
279 file_settings.remove_option(section, option)
281 option_names = ("run_on", )
282 for option in option_names:
283 if file_settings.has_option(section, option):
284 value = file_settings.get(section, option)
285 self.set_run_on(value, state)
286 file_settings.remove_option(section, option)
288 option_names = ("verbosity", )
289 for option in option_names:
290 if file_settings.has_option(section, option):
291 value = VerbosityLevel.from_string(
292 file_settings.get(section, option))
293 self.set_general_verbosity(value, state)
294 file_settings.remove_option(section, option)
296 option_names = ("check_interval", )
297 for option in option_names:
298 if file_settings.has_option(section, option):
299 value = int(file_settings.get(section, option))
300 self.set_general_check_interval(value, state)
301 file_settings.remove_option(section, option)
303 section = "configuration"
304 option_names = ("solver_calls", )
305 for option in option_names:
306 if file_settings.has_option(section, option):
307 value = file_settings.getint(section, option)
308 self.set_configurator_solver_calls(value, state)
309 file_settings.remove_option(section, option)
311 option_names = ("number_of_runs", )
312 for option in option_names:
313 if file_settings.has_option(section, option):
314 value = file_settings.getint(section, option)
315 self.set_configurator_number_of_runs(value, state)
316 file_settings.remove_option(section, option)
318 option_name = "max_iterations"
319 if file_settings.has_option(section, option_name):
320 value = file_settings.getint(section, option_name)
321 self.set_configurator_max_iterations(value, state)
322 file_settings.remove_option(section, option_name)
324 section = "smac2"
325 option_names = ("wallclock_time", )
326 for option in option_names:
327 if file_settings.has_option(section, option):
328 value = file_settings.getint(section, option)
329 self.set_smac2_wallclock_time(value, state)
330 file_settings.remove_option(section, option)
332 option_names = ("cpu_time", )
333 for option in option_names:
334 if file_settings.has_option(section, option):
335 value = file_settings.getint(section, option)
336 self.set_smac2_cpu_time(value, state)
337 file_settings.remove_option(section, option)
339 option_names = ("target_cutoff_length", "each_run_cutoff_length")
340 for option in option_names:
341 if file_settings.has_option(section, option):
342 value = file_settings.get(section, option)
343 self.set_smac2_target_cutoff_length(value, state)
344 file_settings.remove_option(section, option)
346 option_names = ("use_cpu_time_in_tunertime", "countSMACTimeAsTunerTime")
347 for option in option_names:
348 if file_settings.has_option(section, option):
349 value = file_settings.getboolean(section, option)
350 self.set_smac2_use_cpu_time_in_tunertime(value, state)
351 file_settings.remove_option(section, option)
353 option_names = ("cli_cores", )
354 for option in option_names:
355 if file_settings.has_option(section, option):
356 value = file_settings.getint(section, option)
357 self.set_smac2_cli_cores(value, state)
358 file_settings.remove_option(section, option)
360 options_names = ("iteration_limit", "numIterations", "numberOfIterations",
361 "max_iterations")
362 for option in options_names:
363 if file_settings.has_option(section, option):
364 value = file_settings.getint(section, option)
365 self.set_smac2_max_iterations(value, state)
366 file_settings.remove_option(section, option)
368 section = "smac3"
370 option_names = ("n_trials", "number_of_trials", "solver_calls")
371 for option in option_names:
372 if file_settings.has_option(section, option):
373 value = file_settings.getint(section, option)
374 self.set_smac3_number_of_trials(value, state)
375 file_settings.remove_option(section, option)
377 options_names = ("facade", "smac_facade", "smac3_facade")
378 for option in options_names:
379 if file_settings.has_option(section, option):
380 value = file_settings.get(section, option)
381 self.set_smac3_smac_facade(value, state)
382 file_settings.remove_option(section, option)
384 option_names = ("max_ratio", "facade_max_ratio", "initial_trials_max_ratio")
385 for option in option_names:
386 if file_settings.has_option(section, option):
387 value = file_settings.getfloat(section, option)
388 self.set_smac3_facade_max_ratio(value, state)
389 file_settings.remove_option(section, option)
391 options_names = ("crash_cost", )
392 for option in options_names:
393 if file_settings.has_option(section, option):
394 value = file_settings.get(section, option)
395 self.set_smac3_crash_cost(value, state)
396 file_settings.remove_option(section, option)
398 options_names = ("termination_cost_threshold", )
399 for option in options_names:
400 if file_settings.has_option(section, option):
401 value = file_settings.get(section, option)
402 self.set_smac3_termination_cost_threshold(value, state)
403 file_settings.remove_option(section, option)
405 options_names = ("walltime_limit", "wallclock_time")
406 for option in options_names:
407 if file_settings.has_option(section, option):
408 value = file_settings.getfloat(section, option)
409 self.set_smac3_walltime_limit(value, state)
410 file_settings.remove_option(section, option)
412 options_names = ("cputime_limit", )
413 for option in options_names:
414 if file_settings.has_option(section, option):
415 value = file_settings.getfloat(section, option)
416 self.set_smac3_cputime_limit(value, state)
417 file_settings.remove_option(section, option)
419 options_names = ("use_default_config", )
420 for option in options_names:
421 if file_settings.has_option(section, option):
422 value = file_settings.getboolean(section, option)
423 self.set_smac3_use_default_config(value, state)
424 file_settings.remove_option(section, option)
426 options_names = ("min_budget", )
427 for option in options_names:
428 if file_settings.has_option(section, option):
429 value = file_settings.getfloat(section, option)
430 self.set_smac3_min_budget(value, state)
431 file_settings.remove_option(section, option)
433 options_names = ("max_budget", )
434 for option in options_names:
435 if file_settings.has_option(section, option):
436 value = file_settings.getfloat(section, option)
437 self.set_smac3_max_budget(value, state)
438 file_settings.remove_option(section, option)
440 section = "irace"
441 option_names = ("max_time", )
442 for option in option_names:
443 if file_settings.has_option(section, option):
444 value = file_settings.getint(section, option)
445 self.set_irace_max_time(value, state)
446 file_settings.remove_option(section, option)
448 option_names = ("max_experiments", )
449 for option in option_names:
450 if file_settings.has_option(section, option):
451 value = file_settings.getint(section, option)
452 self.set_irace_max_experiments(value, state)
453 file_settings.remove_option(section, option)
455 option_names = ("first_test", )
456 for option in option_names:
457 if file_settings.has_option(section, option):
458 value = file_settings.getint(section, option)
459 self.set_irace_first_test(value, state)
460 file_settings.remove_option(section, option)
462 option_names = ("mu", )
463 for option in option_names:
464 if file_settings.has_option(section, option):
465 value = file_settings.getint(section, option)
466 self.set_irace_mu(value, state)
467 file_settings.remove_option(section, option)
469 option_names = ("nb_iterations", "iterations", "max_iterations")
470 for option in option_names:
471 if file_settings.has_option(section, option):
472 value = file_settings.getint(section, option)
473 self.set_irace_max_iterations(value, state)
474 file_settings.remove_option(section, option)
476 section = "slurm"
477 option_names = ("number_of_jobs_in_parallel", "num_job_in_parallel")
478 for option in option_names:
479 if file_settings.has_option(section, option):
480 value = file_settings.getint(section, option)
481 self.set_number_of_jobs_in_parallel(value, state)
482 file_settings.remove_option(section, option)
484 option_names = ("max_parallel_runs_per_node", "clis_per_node")
485 for option in option_names:
486 if file_settings.has_option(section, option):
487 value = file_settings.getint(section, option)
488 self.set_slurm_max_parallel_runs_per_node(value, state)
489 file_settings.remove_option(section, option)
491 section = "ablation"
492 option_names = ("racing", "ablation_racing")
493 for option in option_names:
494 if file_settings.has_option(section, option):
495 value = file_settings.getboolean(section, option)
496 self.set_ablation_racing_flag(value, state)
497 file_settings.remove_option(section, option)
499 section = "parallel_portfolio"
500 option_names = ("check_interval", )
501 for option in option_names:
502 if file_settings.has_option(section, option):
503 value = int(file_settings.get(section, option))
504 self.set_parallel_portfolio_check_interval(value, state)
505 file_settings.remove_option(section, option)
507 option_names = ("num_seeds_per_solver", )
508 for option in option_names:
509 if file_settings.has_option(section, option):
510 value = int(file_settings.get(section, option))
511 self.set_parallel_portfolio_number_of_seeds_per_solver(value, state)
512 file_settings.remove_option(section, option)
514 # TODO: Report on any unknown settings that were read
515 sections = file_settings.sections()
517 for section in sections:
518 for option in file_settings[section]:
519 # TODO: Should check the options are valid Slurm options
520 if section == "slurm":
521 value = file_settings.get(section, option)
522 self.add_slurm_extra_option(option, value, state)
523 else:
524 print(f'Unrecognised section - option combination: "{section} '
525 f'{option}" in file {file_path} ignored')
527 # Print error if unable to read the settings
528 elif Path(file_path).exists():
529 print(f"ERROR: Failed to read settings from {file_path} The file may have "
530 "been empty or be in another format than INI. Default Setting values "
531 "will be used.")
533 def write_used_settings(self: Settings) -> None:
534 """Write the used settings to the default locations."""
535 # Write to latest settings file
536 self.write_settings_ini(self.DEFAULT_settings_dir / "latest.ini")
538 def write_settings_ini(self: Settings, file_path: Path) -> None:
539 """Write the settings to an INI file."""
540 # Create needed directories if they don't exist
541 file_path.parent.mkdir(parents=True, exist_ok=True)
542 slurm_extra_section_options = None
543 if self.__settings.has_section("slurm_extra"):
544 # Slurm extra options are not written as a seperate section
545 slurm_extra_section_options = {}
546 for key in self.__settings["slurm_extra"]:
547 self.__settings["slurm"][key] = self.__settings["slurm_extra"][key]
548 slurm_extra_section_options[key] = self.__settings["slurm_extra"][key]
549 self.__settings.remove_section("slurm_extra")
550 # We do not write None values
551 removed = []
552 for section in self.__settings.sections():
553 for option in self.__settings[section]:
554 try:
555 if ast.literal_eval(str(self.__settings[section][option])) is None:
556 del self.__settings[section][option]
557 removed.append((section, option))
558 except Exception:
559 pass
560 # Write the settings to file
561 with file_path.open("w") as settings_file:
562 self.__settings.write(settings_file)
563 # Rebuild slurm extra if needed
564 if slurm_extra_section_options is not None:
565 self.__settings.add_section("slurm_extra")
566 for key in slurm_extra_section_options:
567 self.__settings["slurm_extra"][key] = slurm_extra_section_options[key]
568 # Rebuild None if needed
569 for section, option in removed:
570 self.__settings[section][option] = "None"
572 def __init_section(self: Settings, section: str) -> None:
573 if section not in self.__settings:
574 self.__settings[section] = {}
576 @staticmethod
577 def __check_setting_state(current_state: SettingState,
578 new_state: SettingState, name: str) -> bool:
579 change_setting_ok = True
581 if current_state == SettingState.FILE and new_state == SettingState.DEFAULT:
582 change_setting_ok = False
583 print(f"Warning: Attempting to overwrite setting for {name} with default "
584 "value; keeping the value read from file!")
585 elif (current_state == SettingState.CMD_LINE
586 and new_state == SettingState.DEFAULT):
587 change_setting_ok = False
588 print(f"Warning: Attempting to overwrite setting for {name} with default "
589 "value; keeping the value read from command line!")
590 elif current_state == SettingState.CMD_LINE and new_state == SettingState.FILE:
591 change_setting_ok = False
592 print(f"Warning: Attempting to overwrite setting for {name} with value from "
593 "file; keeping the value read from command line!")
595 return change_setting_ok
597 # General settings ###
598 def set_general_sparkle_objectives(
599 self: Settings,
600 value: list[SparkleObjective] = [DEFAULT_general_sparkle_objective, ],
601 origin: SettingState = SettingState.DEFAULT) -> None:
602 """Set the sparkle objective."""
603 section = "general"
604 name = "objectives"
606 if value is not None and self.__check_setting_state(
607 self.__general_sparkle_objective_set, origin, name):
608 if isinstance(value, list):
609 value = ",".join([str(obj) for obj in value])
610 else:
611 value = str(value)
613 # Append standard Sparkle Objectives
614 if "status" not in value:
615 value += ",status:metric"
616 if "cpu_time" not in value:
617 value += ",cpu_time:metric"
618 if "wall_time" not in value:
619 value += ",wall_time:metric"
620 if "memory" not in value:
621 value += ",memory:metric"
623 self.__init_section(section)
624 self.__general_sparkle_objective_set = origin
625 self.__settings[section][name] = value
627 def get_general_sparkle_objectives(
628 self: Settings,
629 filter_metric: bool = False) -> list[SparkleObjective]:
630 """Return the Sparkle objectives."""
631 if self.__general_sparkle_objective_set == SettingState.NOT_SET:
632 self.set_general_sparkle_objectives()
634 objectives = [resolve_objective(obj)
635 for obj in self.__settings["general"]["objectives"].split(",")]
637 if filter_metric:
638 return [obj for obj in objectives if not obj.metric]
640 return objectives
642 def set_general_sparkle_configurator(
643 self: Settings,
644 value: str = DEFAULT_general_sparkle_configurator,
645 origin: SettingState = SettingState.DEFAULT) -> None:
646 """Set the Sparkle configurator."""
647 section = "general"
648 name = "configurator"
649 if value is not None and self.__check_setting_state(
650 self.__general_sparkle_configurator_set, origin, name):
651 self.__init_section(section)
652 self.__general_sparkle_configurator_set = origin
653 self.__settings[section][name] = value
655 def get_general_sparkle_configurator(self: Settings) -> Configurator:
656 """Return the configurator init method."""
657 if self.__general_sparkle_configurator_set == SettingState.NOT_SET:
658 self.set_general_sparkle_configurator()
659 configurator_var = self.__settings["general"]["configurator"]
660 if (self.__general_sparkle_configurator is None
661 or self.__general_sparkle_configurator.name != configurator_var):
662 configurator_subclass =\
663 cim.resolve_configurator(self.__settings["general"]["configurator"])
664 if configurator_subclass is not None:
665 self.__general_sparkle_configurator = configurator_subclass(
666 base_dir=Path(),
667 output_path=Settings.DEFAULT_configuration_output_raw)
668 else:
669 print("WARNING: Configurator class name not recognised: "
670 f'{self.__settings["general"]["configurator"]}. '
671 "Configurator not set.")
672 return self.__general_sparkle_configurator
674 def set_general_sparkle_selector(
675 self: Settings,
676 value: Path = DEFAULT_general_sparkle_selector,
677 origin: SettingState = SettingState.DEFAULT) -> None:
678 """Set the Sparkle selector."""
679 section = "general"
680 name = "selector"
681 if value is not None and self.__check_setting_state(
682 self.__general_sparkle_selector_set, origin, name):
683 self.__init_section(section)
684 self.__general_sparkle_selector_set = origin
685 self.__settings[section][name] = str(value)
687 def get_general_sparkle_selector(self: Settings) -> Selector:
688 """Return the selector init method."""
689 if self.__general_sparkle_selector_set == SettingState.NOT_SET:
690 self.set_general_sparkle_selector()
691 return Selector(Path(self.__settings["general"]["selector"]),
692 self.DEFAULT_selection_output_raw)
694 def set_general_target_cutoff_time(
695 self: Settings, value: int = DEFAULT_general_target_cutoff_time,
696 origin: SettingState = SettingState.DEFAULT) -> None:
697 """Set the cutoff time in seconds for target algorithms."""
698 section = "general"
699 name = "target_cutoff_time"
701 if value is not None and self.__check_setting_state(
702 self.__general_target_cutoff_time_set, origin, name):
703 self.__init_section(section)
704 self.__general_target_cutoff_time_set = origin
705 self.__settings[section][name] = str(value)
707 def get_general_target_cutoff_time(self: Settings) -> int:
708 """Return the cutoff time in seconds for target algorithms."""
709 if self.__general_target_cutoff_time_set == SettingState.NOT_SET:
710 self.set_general_target_cutoff_time()
711 return int(self.__settings["general"]["target_cutoff_time"])
713 def set_general_extractor_cutoff_time(
714 self: Settings, value: int = DEFAULT_general_extractor_cutoff_time,
715 origin: SettingState = SettingState.DEFAULT) -> None:
716 """Set the cutoff time in seconds for feature extraction."""
717 section = "general"
718 name = "extractor_cutoff_time"
720 if value is not None and self.__check_setting_state(
721 self.__general_extractor_cutoff_time_set, origin, name):
722 self.__init_section(section)
723 self.__general_extractor_cutoff_time_set = origin
724 self.__settings[section][name] = str(value)
726 def get_general_extractor_cutoff_time(self: Settings) -> int:
727 """Return the cutoff time in seconds for feature extraction."""
728 if self.__general_extractor_cutoff_time_set == SettingState.NOT_SET:
729 self.set_general_extractor_cutoff_time()
730 return int(self.__settings["general"]["extractor_cutoff_time"])
732 def set_number_of_jobs_in_parallel(
733 self: Settings, value: int = DEFAULT_number_of_jobs_in_parallel,
734 origin: SettingState = SettingState.DEFAULT) -> None:
735 """Set the number of runs Sparkle can do in parallel."""
736 section = "slurm"
737 name = "number_of_jobs_in_parallel"
739 if value is not None and self.__check_setting_state(
740 self.__number_of_jobs_in_parallel_set, origin, name):
741 self.__init_section(section)
742 self.__number_of_jobs_in_parallel_set = origin
743 self.__settings[section][name] = str(value)
745 def get_number_of_jobs_in_parallel(self: Settings) -> int:
746 """Return the number of runs Sparkle can do in parallel."""
747 if self.__number_of_jobs_in_parallel_set == SettingState.NOT_SET:
748 self.set_number_of_jobs_in_parallel()
750 return int(self.__settings["slurm"]["number_of_jobs_in_parallel"])
752 def set_general_verbosity(
753 self: Settings, value: VerbosityLevel = DEFAULT_general_verbosity,
754 origin: SettingState = SettingState.DEFAULT) -> None:
755 """Set the general verbosity to use."""
756 section = "general"
757 name = "verbosity"
759 if value is not None and self.__check_setting_state(
760 self.__general_verbosity_set, origin, name):
761 self.__init_section(section)
762 self.__general_verbosity_set = origin
763 self.__settings[section][name] = value.name
765 def get_general_verbosity(self: Settings) -> VerbosityLevel:
766 """Return the general verbosity."""
767 if self.__general_verbosity_set == SettingState.NOT_SET:
768 self.set_general_verbosity()
770 return VerbosityLevel.from_string(
771 self.__settings["general"]["verbosity"])
773 def set_general_check_interval(
774 self: Settings,
775 value: int = DEFAULT_general_check_interval,
776 origin: SettingState = SettingState.DEFAULT) -> None:
777 """Set the general check interval."""
778 section = "general"
779 name = "check_interval"
781 if value is not None and self.__check_setting_state(
782 self.__general_check_interval_set, origin, name):
783 self.__init_section(section)
784 self.__general_check_interval_set = origin
785 self.__settings[section][name] = str(value)
787 def get_general_check_interval(self: Settings) -> int:
788 """Return the general check interval."""
789 if self.__general_check_interval_set == SettingState.NOT_SET:
790 self.set_general_check_interval()
792 return int(self.__settings["general"]["check_interval"])
794 # Configuration settings General ###
796 def get_configurator_settings(self: Settings,
797 configurator_name: str) -> dict[str, any]:
798 """Return the configurator settings."""
799 configurator_settings = {
800 "number_of_runs": self.get_configurator_number_of_runs(),
801 "solver_calls": self.get_configurator_solver_calls(),
802 "cutoff_time": self.get_general_target_cutoff_time(),
803 "max_iterations": self.get_configurator_max_iterations()
804 }
805 # In the settings below, we default to the configurator general settings if no
806 # specific configurator settings are given, by using the [None] or [Value]
807 if configurator_name == cim.SMAC2.__name__:
808 # Return all settings from the SMAC2 section
809 configurator_settings.update({
810 "cpu_time": self.get_smac2_cpu_time(),
811 "wallclock_time": self.get_smac2_wallclock_time(),
812 "target_cutoff_length": self.get_smac2_target_cutoff_length(),
813 "use_cpu_time_in_tunertime": self.get_smac2_use_cpu_time_in_tunertime(),
814 "cli_cores": self.get_smac2_cli_cores(),
815 "max_iterations": self.get_smac2_max_iterations()
816 or configurator_settings["max_iterations"],
817 })
818 elif configurator_name == cim.SMAC3.__name__:
819 # Return all settings from the SMAC3 section
820 del configurator_settings["max_iterations"] # SMAC3 does not have this?
821 configurator_settings.update({
822 "smac_facade": self.get_smac3_smac_facade(),
823 "max_ratio": self.get_smac3_facade_max_ratio(),
824 "crash_cost": self.get_smac3_crash_cost(),
825 "termination_cost_threshold":
826 self.get_smac3_termination_cost_threshold(),
827 "walltime_limit": self.get_smac3_walltime_limit(),
828 "cputime_limit": self.get_smac3_cputime_limit(),
829 "use_default_config": self.get_smac3_use_default_config(),
830 "min_budget": self.get_smac3_min_budget(),
831 "max_budget": self.get_smac3_max_budget(),
832 "solver_calls": self.get_smac3_number_of_trials()
833 or configurator_settings["solver_calls"],
834 })
835 # Do not pass None values to SMAC3, it Scenario resolves default settings
836 configurator_settings = {key: value
837 for key, value in configurator_settings.items()
838 if value is not None}
839 elif configurator_name == cim.IRACE.__name__:
840 # Return all settings from the IRACE section
841 configurator_settings.update({
842 "solver_calls": self.get_irace_max_experiments(),
843 "max_time": self.get_irace_max_time(),
844 "first_test": self.get_irace_first_test(),
845 "mu": self.get_irace_mu(),
846 "max_iterations": self.get_irace_max_iterations()
847 or configurator_settings["max_iterations"],
848 })
849 if (configurator_settings["solver_calls"] == 0
850 and configurator_settings["max_time"] == 0): # Default to base
851 configurator_settings["solver_calls"] =\
852 self.get_configurator_solver_calls()
853 return configurator_settings
855 def set_configurator_solver_calls(
856 self: Settings, value: int = DEFAULT_configurator_solver_calls,
857 origin: SettingState = SettingState.DEFAULT) -> None:
858 """Set the number of solver calls."""
859 section = "configuration"
860 name = "solver_calls"
862 if value is not None and self.__check_setting_state(
863 self.__config_solver_calls_set, origin, name):
864 self.__init_section(section)
865 self.__config_solver_calls_set = origin
866 self.__settings[section][name] = str(value)
868 def get_configurator_solver_calls(self: Settings) -> int | None:
869 """Return the maximum number of solver calls the configurator can do."""
870 if self.__config_solver_calls_set == SettingState.NOT_SET:
871 self.set_configurator_solver_calls()
873 return int(self.__settings["configuration"]["solver_calls"])
875 def set_configurator_number_of_runs(
876 self: Settings, value: int = DEFAULT_configurator_number_of_runs,
877 origin: SettingState = SettingState.DEFAULT) -> None:
878 """Set the number of configuration runs."""
879 section = "configuration"
880 name = "number_of_runs"
882 if value is not None and self.__check_setting_state(
883 self.__config_number_of_runs_set, origin, name):
884 self.__init_section(section)
885 self.__config_number_of_runs_set = origin
886 self.__settings[section][name] = str(value)
888 def get_configurator_number_of_runs(self: Settings) -> int:
889 """Return the number of configuration runs."""
890 if self.__config_number_of_runs_set == SettingState.NOT_SET:
891 self.set_configurator_number_of_runs()
893 return int(self.__settings["configuration"]["number_of_runs"])
895 def set_configurator_max_iterations(
896 self: Settings, value: int = DEFAULT_configurator_maximum_iterations,
897 origin: SettingState = SettingState.DEFAULT) -> None:
898 """Set the number of configuration runs."""
899 section = "configuration"
900 name = "max_iterations"
902 if self.__check_setting_state(
903 self.__config_max_iterations_set, origin, name):
904 self.__init_section(section)
905 self.__config_max_iterations_set = origin
906 self.__settings[section][name] = str(value)
908 def get_configurator_max_iterations(self: Settings) -> int | None:
909 """Get the maximum number of configurator iterations."""
910 if self.__config_max_iterations_set == SettingState.NOT_SET:
911 self.set_configurator_max_iterations()
912 max_iterations = self.__settings["configuration"]["max_iterations"]
913 return int(max_iterations) if max_iterations.isdigit() else None
915 # Configuration: SMAC2 specific settings ###
917 def set_smac2_wallclock_time(
918 self: Settings, value: int = DEFAULT_smac2_wallclock_time,
919 origin: SettingState = SettingState.DEFAULT) -> None:
920 """Set the budget per configuration run in seconds (wallclock)."""
921 section = "smac2"
922 name = "wallclock_time"
924 if self.__check_setting_state(
925 self.__smac2_wallclock_time_set, origin, name):
926 self.__init_section(section)
927 self.__smac2_wallclock_time_set = origin
928 self.__settings[section][name] = str(value)
930 def get_smac2_wallclock_time(self: Settings) -> int | None:
931 """Return the budget per configuration run in seconds (wallclock)."""
932 if self.__smac2_wallclock_time_set == SettingState.NOT_SET:
933 self.set_smac2_wallclock_time()
934 wallclock_time = self.__settings["smac2"]["wallclock_time"]
935 return int(wallclock_time) if wallclock_time.isdigit() else None
937 def set_smac2_cpu_time(
938 self: Settings, value: int = DEFAULT_smac2_cpu_time,
939 origin: SettingState = SettingState.DEFAULT) -> None:
940 """Set the budget per configuration run in seconds (cpu)."""
941 section = "smac2"
942 name = "cpu_time"
944 if self.__check_setting_state(
945 self.__smac2_cpu_time_set, origin, name):
946 self.__init_section(section)
947 self.__smac2_cpu_time_set = origin
948 self.__settings[section][name] = str(value)
950 def get_smac2_cpu_time(self: Settings) -> int | None:
951 """Return the budget per configuration run in seconds (cpu)."""
952 if self.__smac2_cpu_time_set == SettingState.NOT_SET:
953 self.set_smac2_cpu_time()
954 cpu_time = self.__settings["smac2"]["cpu_time"]
955 return int(cpu_time) if cpu_time.isdigit() else None
957 def set_smac2_target_cutoff_length(
958 self: Settings, value: str = DEFAULT_smac2_target_cutoff_length,
959 origin: SettingState = SettingState.DEFAULT) -> None:
960 """Set the target algorithm cutoff length."""
961 section = "smac2"
962 name = "target_cutoff_length"
964 if value is not None and self.__check_setting_state(
965 self.__smac2_target_cutoff_length_set, origin, name):
966 self.__init_section(section)
967 self.__smac2_target_cutoff_length_set = origin
968 self.__settings[section][name] = str(value)
970 def get_smac2_target_cutoff_length(self: Settings) -> str:
971 """Return the target algorithm cutoff length.
973 'A domain specific measure of when the algorithm should consider itself done.'
975 Returns:
976 The target algorithm cutoff length.
977 """
978 if self.__smac2_target_cutoff_length_set == SettingState.NOT_SET:
979 self.set_smac2_target_cutoff_length()
980 return self.__settings["smac2"]["target_cutoff_length"]
982 def set_smac2_use_cpu_time_in_tunertime(
983 self: Settings, value: bool = DEFAULT_smac2_use_cpu_time_in_tunertime,
984 origin: SettingState = SettingState.DEFAULT) -> None:
985 """Set whether to use CPU time in tunertime."""
986 section = "smac2"
987 name = "use_cpu_time_in_tunertime"
989 if self.__check_setting_state(
990 self.__smac2_use_cpu_time_in_tunertime_set, origin, name):
991 self.__init_section(section)
992 self.__smac2_use_cpu_time_in_tunertime_set = origin
993 self.__settings[section][name] = str(value)
995 def get_smac2_use_cpu_time_in_tunertime(self: Settings) -> bool:
996 """Return whether to use CPU time in tunertime."""
997 if self.__smac2_use_cpu_time_in_tunertime_set == SettingState.NOT_SET:
998 self.set_smac2_use_cpu_time_in_tunertime()
999 return ast.literal_eval(self.__settings["smac2"]["use_cpu_time_in_tunertime"])
1001 def set_smac2_cli_cores(
1002 self: Settings, value: int = DEFAULT_smac2_cli_cores,
1003 origin: SettingState = SettingState.DEFAULT) -> None:
1004 """Set the number of cores to use for SMAC2 CLI."""
1005 section = "smac2"
1006 name = "cli_cores"
1008 if self.__check_setting_state(
1009 self.__smac2_cli_cores_set, origin, name):
1010 self.__init_section(section)
1011 self.__smac2_cli_cores_set = origin
1012 self.__settings[section][name] = str(value)
1014 def get_smac2_cli_cores(self: Settings) -> int | None:
1015 """Number of cores to use to execute runs.
1017 In other words, the number of requests to run at a given time.
1018 """
1019 if self.__smac2_cli_cores_set == SettingState.NOT_SET:
1020 self.set_smac2_cli_cores()
1021 cli_cores = self.__settings["smac2"]["cli_cores"]
1022 return int(cli_cores) if cli_cores.isdigit() else None
1024 def set_smac2_max_iterations(
1025 self: Settings, value: int = DEFAULT_smac2_max_iterations,
1026 origin: SettingState = SettingState.DEFAULT) -> None:
1027 """Set the maximum number of SMAC2 iterations."""
1028 section = "smac2"
1029 name = "max_iterations"
1031 if self.__check_setting_state(
1032 self.__smac2_max_iterations_set, origin, name):
1033 self.__init_section(section)
1034 self.__smac2_max_iterations_set = origin
1035 self.__settings[section][name] = str(value)
1037 def get_smac2_max_iterations(self: Settings) -> int | None:
1038 """Get the maximum number of SMAC2 iterations."""
1039 if self.__smac2_max_iterations_set == SettingState.NOT_SET:
1040 self.set_smac2_max_iterations()
1041 max_iterations = self.__settings["smac2"]["max_iterations"]
1042 return int(max_iterations) if max_iterations.isdigit() else None
1044 # Configuration: SMAC3 specific settings ###
1046 def set_smac3_number_of_trials(
1047 self: Settings, value: int = DEFAULT_smac3_number_of_runs,
1048 origin: SettingState = SettingState.DEFAULT) -> None:
1049 """Set the number of SMAC3 trials."""
1050 section = "smac3"
1051 name = "number_of_runs"
1053 if self.__check_setting_state(
1054 self.__smac3_number_of_trials_set, origin, name):
1055 self.__init_section(section)
1056 self.__smac3_number_of_trials_set = origin
1057 self.__settings[section][name] = str(value)
1059 def get_smac3_number_of_trials(self: Settings) -> int | None:
1060 """Return the number of SMAC3 trials (Solver calls).
1062 'The maximum number of trials (combination of configuration, seed, budget,
1063 and instance, depending on the task) to run.'
1064 """
1065 if self.__smac3_number_of_trials_set == SettingState.NOT_SET:
1066 self.set_smac3_number_of_trials()
1067 number_of_runs = self.__settings["smac3"]["number_of_runs"]
1068 return int(number_of_runs) if number_of_runs.isdigit() else None
1070 def set_smac3_smac_facade(
1071 self: Settings, value: str = DEFAULT_smac3_facade,
1072 origin: SettingState = SettingState.DEFAULT) -> None:
1073 """Set the SMAC3 facade."""
1074 section = "smac3"
1075 name = "facade"
1077 if self.__check_setting_state(self.__smac3_smac_facade_set, origin, name):
1078 self.__init_section(section)
1079 self.__smac3_smac_facade_set = origin
1080 self.__settings[section][name] = str(value)
1082 def get_smac3_smac_facade(self: Settings) -> str:
1083 """Return the SMAC3 facade."""
1084 if self.__smac3_smac_facade_set == SettingState.NOT_SET:
1085 self.set_smac3_smac_facade()
1086 return self.__settings["smac3"]["facade"]
1088 def set_smac3_facade_max_ratio(
1089 self: Settings, value: float = DEFAULT_smac3_facade_max_ratio,
1090 origin: SettingState = SettingState.DEFAULT) -> None:
1091 """Set the SMAC3 facade max ratio."""
1092 section = "smac3"
1093 name = "facade_max_ratio"
1095 if self.__check_setting_state(
1096 self.__smac3_facade_max_ratio_set, origin, name):
1097 self.__init_section(section)
1098 self.__smac3_facade_max_ratio_set = origin
1099 self.__settings[section][name] = str(value)
1101 def get_smac3_facade_max_ratio(self: Settings) -> float:
1102 """Return the SMAC3 facade max ratio."""
1103 if self.__smac3_facade_max_ratio_set == SettingState.NOT_SET:
1104 self.set_smac3_facade_max_ratio()
1105 return ast.literal_eval(self.__settings["smac3"]["facade_max_ratio"])
1107 def set_smac3_crash_cost(self: Settings, value: float = DEFAULT_smac3_crash_cost,
1108 origin: SettingState = SettingState.DEFAULT) -> None:
1109 """Set the SMAC3 objective crash cost."""
1110 section = "smac3"
1111 name = "crash_cost"
1113 if self.__check_setting_state(self.__smac3_crash_cost_set, origin, name):
1114 self.__init_section(section)
1115 self.__smac3_smac_facade_set = origin
1116 self.__settings[section][name] = str(value)
1118 def get_smac3_crash_cost(self: Settings) -> float | list[float]:
1119 """Get the SMAC3 objective crash cost.
1121 'crash_cost : float | list[float], defaults to np.inf
1122 Defines the cost for a failed trial. In case of multi-objective,
1123 each objective can be associated with a different cost.'
1124 """
1125 if self.__smac3_crash_cost_set == SettingState.NOT_SET:
1126 self.set_smac3_crash_cost()
1127 return ast.literal_eval(self.__settings["smac3"]["crash_cost"])
1129 def set_smac3_termination_cost_threshold(
1130 self: Settings,
1131 value: float = DEFAULT_smac3_termination_cost_threshold,
1132 origin: SettingState = SettingState.DEFAULT) -> None:
1133 """Set the SMAC3 termination cost threshold."""
1134 section = "smac3"
1135 name = "termination_cost_threshold"
1137 if self.__check_setting_state(
1138 self.__smac3_termination_cost_threshold_set, origin, name):
1139 self.__init_section(section)
1140 self.__smac3_termination_cost_threshold_set = origin
1141 self.__settings[section][name] = str(value)
1143 def get_smac3_termination_cost_threshold(self: Settings) -> float | list[float]:
1144 """Get the SMAC3 termination cost threshold.
1146 'Defines a cost threshold when the optimization should stop. In case of
1147 multi-objective, each objective *must* be associated with a cost.
1148 The optimization stops when all objectives crossed the threshold.'
1149 """
1150 if self.__smac3_termination_cost_threshold_set == SettingState.NOT_SET:
1151 self.set_smac3_termination_cost_threshold()
1152 return ast.literal_eval(self.__settings["smac3"]["termination_cost_threshold"])
1154 def set_smac3_walltime_limit(
1155 self: Settings, value: float = DEFAULT_smac3_walltime_limit,
1156 origin: SettingState = SettingState.DEFAULT) -> None:
1157 """Set the SMAC3 walltime limit."""
1158 section = "smac3"
1159 name = "walltime_limit"
1161 if self.__check_setting_state(self.__smac3_walltime_limit_set, origin, name):
1162 self.__init_section(section)
1163 self.__smac3_walltime_limit_set = origin
1164 self.__settings[section][name] = str(value)
1166 def get_smac3_walltime_limit(self: Settings) -> float:
1167 """Get the SMAC3 walltime limit.
1169 'The maximum time in seconds that SMAC is allowed to run.'
1170 """
1171 if self.__smac3_walltime_limit_set == SettingState.NOT_SET:
1172 self.set_smac3_walltime_limit()
1173 return ast.literal_eval(self.__settings["smac3"]["walltime_limit"])
1175 def set_smac3_cputime_limit(
1176 self: Settings, value: float = DEFAULT_smac3_cputime_limit,
1177 origin: SettingState = SettingState.DEFAULT) -> None:
1178 """Set the SMAC3 CPU time limit."""
1179 section = "smac3"
1180 name = "cputime_limit"
1182 if self.__check_setting_state(self.__smac3_cputime_limit_set, origin, name):
1183 self.__init_section(section)
1184 self.__smac3_cputime_limit_set = origin
1185 self.__settings[section][name] = str(value)
1187 def get_smac3_cputime_limit(self: Settings) -> float:
1188 """Get the SMAC3 CPU time limit.
1190 'The maximum CPU time in seconds that SMAC is allowed to run.'
1191 """
1192 if self.__smac3_cputime_limit_set == SettingState.NOT_SET:
1193 self.set_smac3_cputime_limit()
1194 return ast.literal_eval(self.__settings["smac3"]["cputime_limit"])
1196 def set_smac3_use_default_config(
1197 self: Settings, value: bool = DEFAULT_smac3_use_default_config,
1198 origin: SettingState = SettingState.DEFAULT) -> None:
1199 """Set the SMAC3 to use default config."""
1200 section = "smac3"
1201 name = "use_default_config"
1203 if self.__check_setting_state(self.__smac3_use_default_config_set, origin, name):
1204 self.__init_section(section)
1205 self.__smac3_use_default_config_set = origin
1206 self.__settings[section][name] = str(value)
1208 def get_smac3_use_default_config(self: Settings) -> bool:
1209 """Get the SMAC3 to use default config.
1211 'If True, the configspace's default configuration is evaluated in the
1212 initial design. For historic benchmark reasons, this is False by default.
1213 Notice, that this will result in n_configs + 1 for the initial design.
1214 Respecting n_trials, this will result in one fewer evaluated
1215 configuration in the optimization.'
1216 """
1217 if self.__smac3_use_default_config_set == SettingState.NOT_SET:
1218 self.set_smac3_use_default_config()
1219 return ast.literal_eval(self.__settings["smac3"]["use_default_config"])
1221 def set_smac3_min_budget(
1222 self: Settings, value: int | float = DEFAULT_smac3_min_budget,
1223 origin: SettingState = SettingState.DEFAULT) -> None:
1224 """Set the SMAC3 min budget."""
1225 section = "smac3"
1226 name = "min_budget"
1228 if self.__check_setting_state(self.__smac3_min_budget_set, origin, name):
1229 self.__init_section(section)
1230 self.__smac3_min_budget_set = origin
1231 self.__settings[section][name] = str(value)
1233 def get_smac3_min_budget(self: Settings) -> int | float:
1234 """Get the SMAC3 min budget.
1236 'The minimum budget (epochs, subset size, number of instances, ...) that
1237 is used for the optimization. Use this argument if you use multi-fidelity
1238 or instance optimization.'
1239 """
1240 if self.__smac3_min_budget_set == SettingState.NOT_SET:
1241 self.set_smac3_min_budget()
1242 return ast.literal_eval(self.__settings["smac3"]["min_budget"])
1244 def set_smac3_max_budget(
1245 self: Settings, value: int | float = DEFAULT_smac3_max_budget,
1246 origin: SettingState = SettingState.DEFAULT) -> None:
1247 """Set the SMAC3 max budget."""
1248 section = "smac3"
1249 name = "max_budget"
1251 if self.__check_setting_state(self.__smac3_max_budget_set, origin, name):
1252 self.__init_section(section)
1253 self.__smac3_max_budget_set = origin
1254 self.__settings[section][name] = str(value)
1256 def get_smac3_max_budget(self: Settings) -> int | float:
1257 """Get the SMAC3 max budget.
1259 'The maximum budget (epochs, subset size, number of instances, ...) that
1260 is used for the optimization. Use this argument if you use multi-fidelity
1261 or instance optimization.'
1262 """
1263 if self.__smac3_max_budget_set == SettingState.NOT_SET:
1264 self.set_smac3_max_budget()
1265 return ast.literal_eval(self.__settings["smac3"]["max_budget"])
1267 # Configuration: IRACE specific settings ###
1269 def get_irace_max_time(self: Settings) -> int:
1270 """Return the max time in seconds for IRACE."""
1271 if self.__irace_max_time_set == SettingState.NOT_SET:
1272 self.set_irace_max_time()
1273 return int(self.__settings["irace"]["max_time"])
1275 def set_irace_max_time(
1276 self: Settings, value: int = DEFAULT_irace_max_time,
1277 origin: SettingState = SettingState.DEFAULT) -> None:
1278 """Set the max time in seconds for IRACE."""
1279 section = "irace"
1280 name = "max_time"
1282 if value is not None and self.__check_setting_state(
1283 self.__irace_max_time_set, origin, name):
1284 self.__init_section(section)
1285 self.__irace_max_time_set = origin
1286 self.__settings[section][name] = str(value)
1288 def get_irace_max_experiments(self: Settings) -> int:
1289 """Return the max number of experiments for IRACE."""
1290 if self.__irace_max_experiments_set == SettingState.NOT_SET:
1291 self.set_irace_max_experiments()
1292 return int(self.__settings["irace"]["max_experiments"])
1294 def set_irace_max_experiments(
1295 self: Settings, value: int = DEFAULT_irace_max_experiments,
1296 origin: SettingState = SettingState.DEFAULT) -> None:
1297 """Set the max number of experiments for IRACE."""
1298 section = "irace"
1299 name = "max_experiments"
1301 if value is not None and self.__check_setting_state(
1302 self.__irace_max_experiments_set, origin, name):
1303 self.__init_section(section)
1304 self.__irace_max_experiments_set = origin
1305 self.__settings[section][name] = str(value)
1307 def get_irace_first_test(self: Settings) -> int | None:
1308 """Return the first test for IRACE.
1310 Specifies how many instances are evaluated before the first
1311 elimination test. IRACE Default: 5. [firstTest]
1312 """
1313 if self.__irace_first_test_set == SettingState.NOT_SET:
1314 self.set_irace_first_test()
1315 first_test = self.__settings["irace"]["first_test"]
1316 return int(first_test) if first_test.isdigit() else None
1318 def set_irace_first_test(
1319 self: Settings, value: int = DEFAULT_irace_first_test,
1320 origin: SettingState = SettingState.DEFAULT) -> None:
1321 """Set the first test for IRACE."""
1322 section = "irace"
1323 name = "first_test"
1325 if self.__check_setting_state(
1326 self.__irace_first_test_set, origin, name):
1327 self.__init_section(section)
1328 self.__irace_first_test_set = origin
1329 self.__settings[section][name] = str(value)
1331 def get_irace_mu(self: Settings) -> int | None:
1332 """Return the mu for IRACE.
1334 Parameter used to define the number of configurations sampled and
1335 evaluated at each iteration. IRACE Default: 5. [mu]
1336 """
1337 if self.__irace_mu_set == SettingState.NOT_SET:
1338 self.set_irace_mu()
1339 mu = self.__settings["irace"]["mu"]
1340 return int(mu) if mu.isdigit() else None
1342 def set_irace_mu(
1343 self: Settings, value: int = DEFAULT_irace_mu,
1344 origin: SettingState = SettingState.DEFAULT) -> None:
1345 """Set the mu for IRACE."""
1346 section = "irace"
1347 name = "mu"
1349 if self.__check_setting_state(
1350 self.__irace_mu_set, origin, name):
1351 self.__init_section(section)
1352 self.__irace_mu_set = origin
1353 self.__settings[section][name] = str(value)
1355 def get_irace_max_iterations(self: Settings) -> int:
1356 """Return the number of iterations for IRACE."""
1357 if self.__irace_max_iterations_set == SettingState.NOT_SET:
1358 self.set_irace_max_iterations()
1359 max_iterations = self.__settings["irace"]["max_iterations"]
1360 return int(max_iterations) if max_iterations.isdigit() else None
1362 def set_irace_max_iterations(
1363 self: Settings, value: int = DEFAULT_irace_max_iterations,
1364 origin: SettingState = SettingState.DEFAULT) -> None:
1365 """Set the number of iterations for IRACE.
1367 Maximum number of iterations to be executed. Each iteration involves the
1368 generation of new configurations and the use of racing to select the best
1369 configurations. By default (with 0), irace calculates a minimum number of
1370 iterations as N^iter = ⌊2 + log2 N param⌋, where N^param is the number of
1371 non-fixed parameters to be tuned.
1372 Setting this parameter may make irace stop sooner than it should without using
1373 all the available budget. IRACE recommends to use the default value (Empty).
1374 """
1375 section = "irace"
1376 name = "max_iterations"
1378 if self.__check_setting_state(
1379 self.__irace_max_iterations_set, origin, name):
1380 self.__init_section(section)
1381 self.__irace_max_iterations_set = origin
1382 self.__settings[section][name] = str(value)
1384 # Slurm settings ###
1386 def set_slurm_max_parallel_runs_per_node(
1387 self: Settings,
1388 value: int = DEFAULT_slurm_max_parallel_runs_per_node,
1389 origin: SettingState = SettingState.DEFAULT) -> None:
1390 """Set the number of algorithms Slurm can run in parallel per node."""
1391 section = "slurm"
1392 name = "max_parallel_runs_per_node"
1394 if value is not None and self.__check_setting_state(
1395 self.__slurm_max_parallel_runs_per_node_set, origin, name):
1396 self.__init_section(section)
1397 self.__slurm_max_parallel_runs_per_node_set = origin
1398 self.__settings[section][name] = str(value)
1400 def get_slurm_max_parallel_runs_per_node(self: Settings) -> int:
1401 """Return the number of algorithms Slurm can run in parallel per node."""
1402 if self.__slurm_max_parallel_runs_per_node_set == SettingState.NOT_SET:
1403 self.set_slurm_max_parallel_runs_per_node()
1405 return int(self.__settings["slurm"]["max_parallel_runs_per_node"])
1407 # SLURM extra options
1409 def add_slurm_extra_option(self: Settings, name: str, value: str,
1410 origin: SettingState = SettingState.DEFAULT) -> None:
1411 """Add additional Slurm options."""
1412 section = "slurm_extra"
1414 current_state = (self.__slurm_extra_options_set[name]
1415 if name in self.__slurm_extra_options_set
1416 else SettingState.NOT_SET)
1418 if value is not None and self.__check_setting_state(current_state, origin, name):
1419 self.__init_section(section)
1420 self.__slurm_extra_options_set[name] = origin
1421 self.__settings[section][name] = str(value)
1423 def get_slurm_extra_options(self: Settings,
1424 as_args: bool = False) -> dict | list:
1425 """Return a dict with additional Slurm options."""
1426 section = "slurm_extra"
1427 options = dict()
1429 if "slurm_extra" in self.__settings.sections():
1430 for option in self.__settings["slurm_extra"]:
1431 options[option] = self.__settings.get(section, option)
1432 if as_args:
1433 return [f"--{key}={options[key]}" for key in options.keys()]
1434 return options
1436 # Ablation settings ###
1438 def set_ablation_racing_flag(self: Settings, value: bool = DEFAULT_ablation_racing,
1439 origin: SettingState = SettingState.DEFAULT) -> None:
1440 """Set a flag indicating whether racing should be used for ablation."""
1441 section = "ablation"
1442 name = "racing"
1444 if value is not None and self.__check_setting_state(
1445 self.__ablation_racing_flag_set, origin, name):
1446 self.__init_section(section)
1447 self.__ablation_racing_flag_set = origin
1448 self.__settings[section][name] = str(value)
1450 def get_ablation_racing_flag(self: Settings) -> bool:
1451 """Return a bool indicating whether the racing flag is set for ablation."""
1452 if self.__ablation_racing_flag_set == SettingState.NOT_SET:
1453 self.set_ablation_racing_flag()
1455 return bool(self.__settings["ablation"]["racing"])
1457 # Parallel Portfolio settings
1459 def set_parallel_portfolio_check_interval(
1460 self: Settings,
1461 value: int = DEFAULT_parallel_portfolio_check_interval,
1462 origin: SettingState = SettingState.DEFAULT) -> None:
1463 """Set the parallel portfolio check interval."""
1464 section = "parallel_portfolio"
1465 name = "check_interval"
1467 if value is not None and self.__check_setting_state(
1468 self.__parallel_portfolio_check_interval_set, origin, name):
1469 self.__init_section(section)
1470 self.__parallel_portfolio_check_interval_set = origin
1471 self.__settings[section][name] = str(value)
1473 def get_parallel_portfolio_check_interval(self: Settings) -> int:
1474 """Return the parallel portfolio check interval."""
1475 if self.__parallel_portfolio_check_interval_set == SettingState.NOT_SET:
1476 self.set_parallel_portfolio_check_interval()
1478 return int(
1479 self.__settings["parallel_portfolio"]["check_interval"])
1481 def set_parallel_portfolio_number_of_seeds_per_solver(
1482 self: Settings,
1483 value: int = DEFAULT_parallel_portfolio_num_seeds_per_solver,
1484 origin: SettingState = SettingState.DEFAULT) -> None:
1485 """Set the parallel portfolio seeds per solver to start."""
1486 section = "parallel_portfolio"
1487 name = "num_seeds_per_solver"
1489 if value is not None and self.__check_setting_state(
1490 self.__parallel_portfolio_num_seeds_per_solver_set, origin, name):
1491 self.__init_section(section)
1492 self.__parallel_portfolio_num_seeds_per_solver_set = origin
1493 self.__settings[section][name] = str(value)
1495 def get_parallel_portfolio_number_of_seeds_per_solver(self: Settings) -> int:
1496 """Return the parallel portfolio seeds per solver to start."""
1497 if self.__parallel_portfolio_num_seeds_per_solver_set == SettingState.NOT_SET:
1498 self.set_parallel_portfolio_number_of_seeds_per_solver()
1500 return int(
1501 self.__settings["parallel_portfolio"]["num_seeds_per_solver"])
1503 def set_run_on(self: Settings, value: Runner = DEFAULT_general_run_on,
1504 origin: SettingState = SettingState.DEFAULT) -> None:
1505 """Set the compute on which to run."""
1506 section = "general"
1507 name = "run_on"
1509 if value is not None and self.__check_setting_state(
1510 self.__run_on_set, origin, name):
1511 self.__init_section(section)
1512 self.__run_on_set = origin
1513 self.__settings[section][name] = value
1515 def get_run_on(self: Settings) -> Runner:
1516 """Return the compute on which to run."""
1517 if self.__run_on_set == SettingState.NOT_SET:
1518 self.set_run_on()
1520 return Runner(self.__settings["general"]["run_on"])
1522 @staticmethod
1523 def check_settings_changes(cur_settings: Settings, prev_settings: Settings) -> bool:
1524 """Check if there are changes between the previous and the current settings.
1526 Prints any section changes, printing None if no setting was found.
1528 Args:
1529 cur_settings: The current settings
1530 prev_settings: The previous settings
1532 Returns:
1533 True iff there are no changes.
1534 """
1535 cur_dict = cur_settings.__settings._sections
1536 prev_dict = prev_settings.__settings._sections
1538 cur_sections_set = set(cur_dict.keys())
1539 prev_sections_set = set(prev_dict.keys())
1540 sections_removed = prev_sections_set - cur_sections_set
1541 if sections_removed:
1542 print("Warning: the following sections have been removed:")
1543 for section in sections_removed:
1544 print(f" - Section '{section}'")
1546 sections_added = cur_sections_set - prev_sections_set
1547 if sections_added:
1548 print("Warning: the following sections have been added:")
1549 for section in sections_added:
1550 print(f" - Section '{section}'")
1552 sections_remained = cur_sections_set & prev_sections_set
1553 option_changed = False
1554 for section in sections_remained:
1555 printed_section = False
1556 names = set(cur_dict[section].keys()) | set(prev_dict[section].keys())
1557 for name in names:
1558 # if name is not present in one of the two dicts, get None as placeholder
1559 cur_val = cur_dict[section].get(name, None)
1560 prev_val = prev_dict[section].get(name, None)
1562 # If cur val is None, it is default
1563 if cur_val is not None and cur_val != prev_val:
1564 # Have we printed the initial warning?
1565 if not option_changed:
1566 print("Warning: The following attributes/options have changed:")
1567 option_changed = True
1569 # do we have yet to print the section?
1570 if not printed_section:
1571 print(f" - In the section '{section}':")
1572 printed_section = True
1574 # print actual change
1575 print(f" · '{name}' changed from '{prev_val}' to '{cur_val}'")
1577 return not (sections_removed or sections_added or option_changed)