Coverage for sparkle/platform/settings_objects.py: 97%

1"""Classes and Enums to control settings.""" 

2 

3from __future__ import annotations 

4import configparser 

5import argparse 

6from enum import Enum 

7from pathlib import Path 

8from typing import Any, NamedTuple, Optional 

9 

10from runrunner import Runner 

11 

12from sparkle.types import SparkleObjective, resolve_objective 

13from sparkle.configurator.configurator import Configurator 

14from sparkle.configurator import implementations as cim 

15from sparkle.platform.cli_types import VerbosityLevel 

16 

17 

18class Option(NamedTuple): 

19 """Class to define an option in the Settings.""" 

20 

21 name: str 

22 section: str 

23 type: Any 

24 default_value: Any 

25 alternatives: tuple[str, ...] 

26 help: str = "" 

27 cli_kwargs: dict[str, Any] = {} 

28 

29 def __str__(self: Option) -> str: 

30 """Return the option name.""" 

31 return self.name 

32 

33 def __eq__(self: Option, other: Any) -> bool: 

34 """Check if two options are equal.""" 

35 if isinstance(other, Option): 

36 return ( 

37 self.name == other.name 

38 and self.section == other.section 

39 and self.type == other.type 

40 and self.default_value == other.default_value 

41 and self.alternatives == other.alternatives 

42 ) 

43 if isinstance(other, str): 

44 return self.name == other or other in self.alternatives 

45 return False 

46 

47 @property 

48 def args(self: Option) -> list[str]: 

49 """Return the option names as a command line arguments.""" 

50 return [ 

51 f"--{name.replace('_', '-')}" 

52 for name in [self.name] + list(self.alternatives) 

53 ] 

54 

55 @property 

56 def kwargs(self: Option) -> dict[str, Any]: 

57 """Return the option attributes as kwargs.""" 

58 kw = {"help": self.help, **self.cli_kwargs} 

59 

60 # If this option uses a boolean flag action, argparse must NOT receive 'type' 

61 action = kw.get("action") 

62 if action in ("store_true", "store_false"): 

63 return kw 

64 

65 # Otherwise include the base 'type' 

66 return {"type": self.type, **kw} 

67 

68 

69class Settings: 

70 """Class to read, write, set, and get settings.""" 

71 

72 # CWD Prefix 

73 cwd_prefix = Path() # Empty for now 

74 

75 # Library prefix 

76 lib_prefix = Path(__file__).parent.parent.resolve() 

77 

78 # Default directory names 

79 rawdata_dir = Path("Raw_Data") 

80 analysis_dir = Path("Analysis") 

81 DEFAULT_settings_dir = Path("Settings") 

82 __settings_file = Path("sparkle_settings.ini") 

83 __latest_settings_file = Path("latest.ini") 

84 

85 # Default settings path 

86 DEFAULT_settings_path = Path(cwd_prefix / DEFAULT_settings_dir / __settings_file) 

87 DEFAULT_previous_settings_path = Path( 

88 cwd_prefix / DEFAULT_settings_dir / __latest_settings_file 

89 ) 

90 DEFAULT_reference_dir = DEFAULT_settings_dir / "Reference_Lists" 

91 

92 # Default library pathing 

93 DEFAULT_components = lib_prefix / "Components" 

94 

95 # Report Component: Bilbiography 

96 bibliography_path = DEFAULT_components / "latex_source" / "report.bib" 

97 

98 # Example settings path 

99 DEFAULT_example_settings_path = Path(DEFAULT_components / "sparkle_settings.ini") 

100 

101 # Runsolver component 

102 DEFAULT_runsolver_dir = DEFAULT_components / "runsolver" / "src" 

103 DEFAULT_runsolver_exec = DEFAULT_runsolver_dir / "runsolver" 

104 

105 # Ablation component 

106 DEFAULT_ablation_dir = DEFAULT_components / "ablationAnalysis-0.9.4" 

107 DEFAULT_ablation_exec = DEFAULT_ablation_dir / "ablationAnalysis" 

108 DEFAULT_ablation_validation_exec = DEFAULT_ablation_dir / "ablationValidation" 

109 

110 # Default input directory pathing 

111 DEFAULT_solver_dir = cwd_prefix / "Solvers" 

112 DEFAULT_instance_dir = cwd_prefix / "Instances" 

113 DEFAULT_extractor_dir = cwd_prefix / "Extractors" 

114 DEFAULT_snapshot_dir = cwd_prefix / "Snapshots" 

115 

116 # Default output directory pathing 

117 DEFAULT_tmp_output = cwd_prefix / "Tmp" 

118 DEFAULT_output = cwd_prefix / "Output" 

119 DEFAULT_configuration_output = DEFAULT_output / "Configuration" 

120 DEFAULT_selection_output = DEFAULT_output / "Selection" 

121 DEFAULT_parallel_portfolio_output = DEFAULT_output / "Parallel_Portfolio" 

122 DEFAULT_ablation_output = DEFAULT_output / "Ablation" 

123 DEFAULT_log_output = DEFAULT_output / "Log" 

124 

125 # Default output subdirs 

126 DEFAULT_output_analysis = DEFAULT_output / analysis_dir 

127 

128 # Old default output dirs which should be part of something else 

129 DEFAULT_feature_data = DEFAULT_output / "Feature_Data" 

130 DEFAULT_performance_data = DEFAULT_output / "Performance_Data" 

131 

132 # Collection of all working dirs for platform 

133 DEFAULT_working_dirs = [ 

134 DEFAULT_solver_dir, 

135 DEFAULT_instance_dir, 

136 DEFAULT_extractor_dir, 

137 DEFAULT_output, 

138 DEFAULT_configuration_output, 

139 DEFAULT_selection_output, 

140 DEFAULT_output_analysis, 

141 DEFAULT_tmp_output, 

142 DEFAULT_log_output, 

143 DEFAULT_feature_data, 

144 DEFAULT_performance_data, 

145 DEFAULT_settings_dir, 

146 DEFAULT_reference_dir, 

147 ] 

148 

149 # Old default file paths from GV which should be turned into variables 

150 DEFAULT_feature_data_path = DEFAULT_feature_data / "feature_data.csv" 

151 DEFAULT_performance_data_path = DEFAULT_performance_data / "performance_data.csv" 

152 

153 # Define sections and options 

154 # GENERAL Options 

155 SECTION_general: str = "general" 

156 OPTION_objectives = Option( 

157 "objectives", 

158 SECTION_general, 

159 str, 

160 None, 

161 ("sparkle_objectives",), 

162 "A list of Sparkle objectives.", 

163 cli_kwargs={"nargs": "+"}, 

164 ) 

165 OPTION_configurator = Option( 

166 "configurator", 

167 SECTION_general, 

168 str, 

169 None, 

170 tuple(), 

171 "Name of the configurator to use.", 

172 ) 

173 OPTION_solver_cutoff_time = Option( 

174 "solver_cutoff_time", 

175 SECTION_general, 

176 int, 

177 None, 

178 ("target_cutoff_time", "cutoff_time_each_solver_call"), 

179 "Solver cutoff time in seconds.", 

180 ) 

181 OPTION_extractor_cutoff_time = Option( 

182 "extractor_cutoff_time", 

183 SECTION_general, 

184 int, 

185 None, 

186 tuple("cutoff_time_each_feature_computation"), 

187 "Extractor cutoff time in seconds.", 

188 ) 

189 OPTION_run_on = Option( 

190 "run_on", 

191 SECTION_general, 

192 Runner, 

193 None, 

194 tuple(), 

195 "On which compute resource to execute.", 

196 cli_kwargs={"choices": [Runner.LOCAL, Runner.SLURM]}, 

197 ) 

198 OPTION_verbosity = Option( 

199 "verbosity", 

200 SECTION_general, 

201 VerbosityLevel, 

202 VerbosityLevel.STANDARD, 

203 ("verbosity_level",), 

204 "Verbosity level.", 

205 ) 

206 OPTION_seed = Option( 

207 "seed", 

208 SECTION_general, 

209 int, 

210 None, 

211 tuple(), 

212 "Seed to use for pseudo-random number generators.", 

213 ) 

214 OPTION_appendices = Option( 

215 "appendices", 

216 SECTION_general, 

217 bool, 

218 False, 

219 tuple(), 

220 "Include the appendix section in the generated report.", 

221 cli_kwargs={ 

222 "action": "store_true", 

223 "default": None, 

224 }, 

225 ) 

226 

227 # CONFIGURATION Options 

228 SECTION_configuration = "configuration" 

229 OPTION_configurator_number_of_runs = Option( 

230 "number_of_runs", 

231 SECTION_configuration, 

232 int, 

233 None, 

234 tuple(), 

235 "The number of independent configurator jobs/runs.", 

236 ) 

237 OPTION_configurator_solver_call_budget = Option( 

238 "solver_calls", 

239 SECTION_configuration, 

240 int, 

241 None, 

242 tuple(), 

243 "The maximum number of calls (evaluations) a configurator can do in a single " 

244 "run of the solver.", 

245 ) 

246 OPTION_configurator_max_iterations = Option( 

247 "max_iterations", 

248 SECTION_configuration, 

249 int, 

250 None, 

251 ("maximum_iterations",), 

252 "The maximum number of iterations a configurator can do in a single job.", 

253 ) 

254 

255 # ABLATION Options 

256 SECTION_ablation = "ablation" 

257 OPTION_ablation_racing = Option( 

258 "racing", 

259 SECTION_ablation, 

260 bool, 

261 False, 

262 ("ablation_racing",), 

263 "Set a flag indicating whether racing should be used for ablation.", 

264 ) 

265 OPTION_ablation_clis_per_node = Option( 

266 "clis_per_node", 

267 SECTION_ablation, 

268 int, 

269 None, 

270 ( 

271 "max_parallel_runs_per_node", 

272 "maximum_parallel_runs_per_node", 

273 ), 

274 "The maximum number of ablation analysis jobs to run in parallel on a single " 

275 "node.", 

276 ) 

277 

278 # SELECTION Options 

279 SECTION_selection = "selection" 

280 OPTION_selection_class = Option( 

281 "selector_class", 

282 SECTION_selection, 

283 str, 

284 None, 

285 ("class",), 

286 "Can contain any of the class names as defined in asf.selectors.", 

287 ) 

288 OPTION_selection_model = Option( 

289 "selector_model", 

290 SECTION_selection, 

291 str, 

292 None, 

293 ("model",), 

294 "Can be any of the sklearn.ensemble models.", 

295 ) 

296 OPTION_minimum_marginal_contribution = Option( 

297 "minimum_marginal_contribution", 

298 SECTION_selection, 

299 float, 

300 0.01, 

301 ( 

302 "minimum_marginal_contribution", 

303 "minimum_contribution", 

304 "contribution_threshold", 

305 ), 

306 "The minimum marginal contribution a solver (configuration) must have to be used for the selector.", 

307 ) 

308 

309 # SMAC2 Options 

310 SECTION_smac2 = "smac2" 

311 OPTION_smac2_wallclock_time_budget = Option( 

312 "wallclock_time_budget", 

313 SECTION_smac2, 

314 int, 

315 None, 

316 ("wallclock_time",), 

317 "The wallclock time budget in seconds for each SMAC2 run.", 

318 ) 

319 OPTION_smac2_cpu_time_budget = Option( 

320 "cpu_time_budget", 

321 SECTION_smac2, 

322 int, 

323 None, 

324 ("cpu_time",), 

325 "The cpu time budget in seconds for each SMAC2 run.", 

326 ) 

327 OPTION_smac2_target_cutoff_length = Option( 

328 "target_cutoff_length", 

329 SECTION_smac2, 

330 str, 

331 None, 

332 ("cutoff_length", "solver_cutoff_length"), 

333 "The target cutoff length for SMAC2 solver call.", 

334 ) 

335 OPTION_smac2_count_tuner_time = Option( 

336 "use_cpu_time_in_tunertime", 

337 SECTION_smac2, 

338 bool, 

339 None, 

340 ("countSMACTimeAsTunerTime",), 

341 "Whether to count and deducted SMAC2 CPU time from the CPU time budget.", 

342 ) 

343 OPTION_smac2_cli_cores = Option( 

344 "cli_cores", 

345 SECTION_smac2, 

346 int, 

347 None, 

348 tuple(), 

349 "Number of cores to use to execute SMAC2 runs.", 

350 ) 

351 OPTION_smac2_max_iterations = Option( 

352 "max_iterations", 

353 SECTION_smac2, 

354 int, 

355 None, 

356 ( 

357 "iteration_limit", 

358 "numIterations", 

359 "numberOfIterations", 

360 ), 

361 "The maximum number of iterations SMAC2 configurator can do in a single job.", 

362 ) 

363 

364 # SMAC3 Options 

365 SECTION_smac3 = "smac3" 

366 OPTION_smac3_number_of_trials = Option( 

367 "n_trials", 

368 SECTION_smac3, 

369 int, 

370 None, 

371 ("n_trials", "number_of_trials", "solver_calls"), 

372 "Maximum calls SMAC3 is allowed to make to the Solver in a single run/job.", 

373 ) 

374 OPTION_smac3_facade = Option( 

375 "facade", 

376 SECTION_smac3, 

377 str, 

378 "AlgorithmConfigurationFacade", 

379 ("facade", "smac_facade", "smac3_facade"), 

380 "The SMAC3 Facade to use. See the SMAC3 documentation for more options.", 

381 ) 

382 OPTION_smac3_facade_max_ratio = Option( 

383 "facade_max_ratio", 

384 SECTION_smac3, 

385 float, 

386 None, 

387 ("facade_max_ratio", "smac3_facade_max_ratio", "smac3_facade_max_ratio"), 

388 "The SMAC3 Facade max ratio. See the SMAC3 documentation for more options.", 

389 ) 

390 OPTION_smac3_crash_cost = Option( 

391 "crash_cost", 

392 SECTION_smac3, 

393 float, 

394 None, 

395 tuple(), 

396 "Defines the cost for a failed trial, defaults in SMAC3 to np.inf.", 

397 ) 

398 OPTION_smac3_termination_cost_threshold = Option( 

399 "termination_cost_threshold", 

400 SECTION_smac3, 

401 float, 

402 None, 

403 tuple(), 

404 "Defines a cost threshold when the SMAC3 optimization should stop.", 

405 ) 

406 OPTION_smac3_wallclock_time_budget = Option( 

407 "walltime_limit", 

408 SECTION_smac3, 

409 float, 

410 None, 

411 ("wallclock_time", "wallclock_budget", "wallclock_time_budget"), 

412 "The maximum time in seconds that SMAC3 is allowed to run per job.", 

413 ) 

414 OPTION_smac3_cpu_time_budget = Option( 

415 "cputime_limit", 

416 SECTION_smac3, 

417 float, 

418 None, 

419 ("cpu_time", "cpu_budget", "cpu_time_budget"), 

420 "The maximum CPU time in seconds that SMAC3 is allowed to run per job.", 

421 ) 

422 OPTION_smac3_use_default_config = Option( 

423 "use_default_config", 

424 SECTION_smac3, 

425 bool, 

426 None, 

427 tuple(), 

428 "If True, the configspace's default configuration is evaluated in the initial " 

429 "design. For historic benchmark reasons, this is False by default. Notice, that " 

430 "this will result in n_configs + 1 for the initial design. Respecting n_trials, " 

431 "this will result in one fewer evaluated configuration in the optimization.", 

432 ) 

433 OPTION_smac3_min_budget = Option( 

434 "min_budget", 

435 SECTION_smac3, 

436 float, 

437 None, 

438 ("minimum_budget",), 

439 "The minimum budget (epochs, subset size, number of instances, ...) that is used" 

440 " for the optimization. Use this argument if you use multi-fidelity or instance " 

441 "optimization.", 

442 ) 

443 OPTION_smac3_max_budget = Option( 

444 "max_budget", 

445 SECTION_smac3, 

446 float, 

447 None, 

448 ("maximum_budget",), 

449 "The maximum budget (epochs, subset size, number of instances, ...) that is used" 

450 " for the optimization. Use this argument if you use multi-fidelity or instance " 

451 "optimization.", 

452 ) 

453 

454 # IRACE Options 

455 SECTION_irace = "irace" 

456 OPTION_irace_max_time = Option( 

457 "max_time", 

458 SECTION_irace, 

459 int, 

460 0, 

461 ("maximum_time",), 

462 "The maximum time in seconds for each IRACE run/job.", 

463 ) 

464 OPTION_irace_max_experiments = Option( 

465 "max_experiments", 

466 SECTION_irace, 

467 int, 

468 0, 

469 ("maximum_experiments",), 

470 "The maximum number of experiments for each IRACE run/job.", 

471 ) 

472 OPTION_irace_first_test = Option( 

473 "first_test", 

474 SECTION_irace, 

475 int, 

476 None, 

477 tuple(), 

478 "Specifies how many instances are evaluated before the first elimination test. " 

479 "IRACE Default: 5.", 

480 ) 

481 OPTION_irace_mu = Option( 

482 "mu", 

483 SECTION_irace, 

484 int, 

485 None, 

486 tuple(), 

487 "Parameter used to define the number of configurations sampled and evaluated at " 

488 "each iteration. IRACE Default: 5.", 

489 ) 

490 OPTION_irace_max_iterations = Option( 

491 "max_iterations", 

492 SECTION_irace, 

493 int, 

494 None, 

495 ("nb_iterations", "iterations", "max_iterations"), 

496 "Maximum number of iterations to be executed. Each iteration involves the " 

497 "generation of new configurations and the use of racing to select the best " 

498 "configurations. By default (with 0), irace calculates a minimum number of " 

499 "iterations as N^iter = ⌊2 + log2 N param⌋, where N^param is the number of " 

500 "non-fixed parameters to be tuned. Setting this parameter may make irace stop " 

501 "sooner than it should without using all the available budget. IRACE recommends" 

502 " to use the default value (Empty).", 

503 ) 

504 

505 # ParamILS Options 

506 SECTION_paramils = "paramils" 

507 OPTION_paramils_min_runs = Option( 

508 "min_runs", 

509 SECTION_paramils, 

510 int, 

511 None, 

512 ("minimum_runs",), 

513 "Set the minimum number of runs for ParamILS for each run/job.", 

514 ) 

515 OPTION_paramils_max_runs = Option( 

516 "max_runs", 

517 SECTION_paramils, 

518 int, 

519 None, 

520 ("maximum_runs",), 

521 "Set the maximum number of runs for ParamILS for each run/job.", 

522 ) 

523 OPTION_paramils_cpu_time_budget = Option( 

524 "cputime_budget", 

525 SECTION_paramils, 

526 int, 

527 None, 

528 ( 

529 "cputime_limit", 

530 "cputime_limit", 

531 "tunertime_limit", 

532 "tuner_timeout", 

533 "tunerTimeout", 

534 ), 

535 "The maximum CPU time for each ParamILS run/job.", 

536 ) 

537 OPTION_paramils_random_restart = Option( 

538 "random_restart", 

539 SECTION_paramils, 

540 float, 

541 None, 

542 tuple(), 

543 "Set the random restart chance for ParamILS.", 

544 ) 

545 OPTION_paramils_focused = Option( 

546 "focused_approach", 

547 SECTION_paramils, 

548 bool, 

549 False, 

550 ("focused",), 

551 "Set the focused approach for ParamILS.", 

552 ) 

553 OPTION_paramils_count_tuner_time = Option( 

554 "use_cpu_time_in_tunertime", 

555 SECTION_paramils, 

556 bool, 

557 None, 

558 tuple(), 

559 "Whether to count and deducted ParamILS CPU time from the CPU time budget.", 

560 ) 

561 OPTION_paramils_cli_cores = Option( 

562 "cli_cores", 

563 SECTION_paramils, 

564 int, 

565 None, 

566 tuple(), 

567 "Number of cores to use for ParamILS runs.", 

568 ) 

569 OPTION_paramils_max_iterations = Option( 

570 "max_iterations", 

571 SECTION_paramils, 

572 int, 

573 None, 

574 ( 

575 "iteration_limit", 

576 "numIterations", 

577 "numberOfIterations", 

578 "maximum_iterations", 

579 ), 

580 "The maximum number of ParamILS iterations per run/job.", 

581 ) 

582 OPTION_paramils_number_initial_configurations = Option( 

583 "initial_configurations", 

584 SECTION_paramils, 

585 int, 

586 None, 

587 "The number of initial configurations ParamILS should evaluate.", 

588 ) 

589 

590 SECTION_parallel_portfolio = "parallel_portfolio" 

591 OPTION_parallel_portfolio_check_interval = Option( 

592 "check_interval", 

593 SECTION_parallel_portfolio, 

594 int, 

595 None, 

596 tuple(), 

597 "The interval time in seconds when Solvers are checked for their status.", 

598 ) 

599 OPTION_parallel_portfolio_number_of_seeds_per_solver = Option( 

600 "num_seeds_per_solver", 

601 SECTION_parallel_portfolio, 

602 int, 

603 None, 

604 ("solver_seeds",), 

605 "The number of seeds per solver.", 

606 ) 

607 

608 SECTION_slurm = "slurm" 

609 OPTION_slurm_parallel_jobs = Option( 

610 "number_of_jobs_in_parallel", 

611 SECTION_slurm, 

612 int, 

613 None, 

614 ("num_job_in_parallel",), 

615 "The number of jobs to run in parallel.", 

616 ) 

617 OPTION_slurm_prepend_script = Option( 

618 "prepend_script", 

619 SECTION_slurm, 

620 str, 

621 None, 

622 ("job_prepend", "prepend"), 

623 "Slurm script to prepend to the sbatch.", 

624 ) 

625 

626 sections_options: dict[str, list[Option]] = { 

627 SECTION_general: [ 

628 OPTION_objectives, 

629 OPTION_configurator, 

630 OPTION_solver_cutoff_time, 

631 OPTION_extractor_cutoff_time, 

632 OPTION_run_on, 

633 OPTION_appendices, 

634 OPTION_verbosity, 

635 OPTION_seed, 

636 ], 

637 SECTION_configuration: [ 

638 OPTION_configurator_number_of_runs, 

639 OPTION_configurator_solver_call_budget, 

640 OPTION_configurator_max_iterations, 

641 ], 

642 SECTION_ablation: [ 

643 OPTION_ablation_racing, 

644 OPTION_ablation_clis_per_node, 

645 ], 

646 SECTION_selection: [ 

647 OPTION_selection_class, 

648 OPTION_selection_model, 

649 OPTION_minimum_marginal_contribution, 

650 ], 

651 SECTION_smac2: [ 

652 OPTION_smac2_wallclock_time_budget, 

653 OPTION_smac2_cpu_time_budget, 

654 OPTION_smac2_target_cutoff_length, 

655 OPTION_smac2_count_tuner_time, 

656 OPTION_smac2_cli_cores, 

657 OPTION_smac2_max_iterations, 

658 ], 

659 SECTION_smac3: [ 

660 OPTION_smac3_number_of_trials, 

661 OPTION_smac3_facade, 

662 OPTION_smac3_facade_max_ratio, 

663 OPTION_smac3_crash_cost, 

664 OPTION_smac3_termination_cost_threshold, 

665 OPTION_smac3_wallclock_time_budget, 

666 OPTION_smac3_cpu_time_budget, 

667 OPTION_smac3_use_default_config, 

668 OPTION_smac3_min_budget, 

669 OPTION_smac3_max_budget, 

670 ], 

671 SECTION_irace: [ 

672 OPTION_irace_max_time, 

673 OPTION_irace_max_experiments, 

674 OPTION_irace_first_test, 

675 OPTION_irace_mu, 

676 OPTION_irace_max_iterations, 

677 ], 

678 SECTION_paramils: [ 

679 OPTION_paramils_min_runs, 

680 OPTION_paramils_max_runs, 

681 OPTION_paramils_cpu_time_budget, 

682 OPTION_paramils_random_restart, 

683 OPTION_paramils_focused, 

684 OPTION_paramils_count_tuner_time, 

685 OPTION_paramils_cli_cores, 

686 OPTION_paramils_max_iterations, 

687 OPTION_paramils_number_initial_configurations, 

688 ], 

689 SECTION_parallel_portfolio: [ 

690 OPTION_parallel_portfolio_check_interval, 

691 OPTION_parallel_portfolio_number_of_seeds_per_solver, 

692 ], 

693 SECTION_slurm: [OPTION_slurm_parallel_jobs, OPTION_slurm_prepend_script], 

694 } 

695 

696 def __init__( 

697 self: Settings, file_path: Path, argsv: argparse.Namespace = None 

698 ) -> None: 

699 """Initialise a settings object. 

700 

701 Args: 

702 file_path (Path): Path to the settings file. 

703 argsv: The CLI arguments to process. 

704 """ 

705 # Settings 'dictionary' in configparser format 

706 self.__settings = configparser.ConfigParser() 

707 for section in self.sections_options.keys(): 

708 self.__settings.add_section(section) 

709 self.__settings[section] = {} 

710 

711 # General attributes 

712 self.__sparkle_objectives: list[SparkleObjective] = None 

713 self.__general_sparkle_configurator: Configurator = None 

714 self.__solver_cutoff_time: int = None 

715 self.__extractor_cutoff_time: int = None 

716 self.__run_on: Runner = None 

717 self.__appendices: bool = False 

718 self.__verbosity_level: VerbosityLevel = None 

719 self.__seed: Optional[int] = None 

720 

721 # Configuration attributes 

722 self.__configurator_solver_call_budget: int = None 

723 self.__configurator_number_of_runs: int = None 

724 self.__configurator_max_iterations: int = None 

725 

726 # Ablation attributes 

727 self.__ablation_racing_flag: bool = None 

728 self.__ablation_max_parallel_runs_per_node: int = None 

729 

730 # Selection attributes 

731 self.__selection_model: str = None 

732 self.__selection_class: str = None 

733 self.__minimum_marginal_contribution: float = None 

734 

735 # SMAC2 attributes 

736 self.__smac2_wallclock_time_budget: int = None 

737 self.__smac2_cpu_time_budget: int = None 

738 self.__smac2_target_cutoff_length: str = None 

739 self.__smac2_use_tunertime_in_cpu_time_budget: bool = None 

740 self.__smac2_cli_cores: int = None 

741 self.__smac2_max_iterations: int = None 

742 

743 # SMAC3 attributes 

744 self.__smac3_number_of_trials: int = None 

745 self.__smac3_facade: str = None 

746 self.__smac3_facade_max_ratio: float = None 

747 self.__smac3_crash_cost: float = None 

748 self.__smac3_termination_cost_threshold: float = None 

749 self.__smac3_wallclock_time_limit: int = None 

750 self.__smac3_cputime_limit: int = None 

751 self.__smac3_use_default_config: bool = None 

752 self.__smac3_min_budget: float = None 

753 self.__smac3_max_budget: float = None 

754 

755 # IRACE attributes 

756 self.__irace_max_time: int = None 

757 self.__irace_max_experiments: int = None 

758 self.__irace_first_test: int = None 

759 self.__irace_mu: int = None 

760 self.__irace_max_iterations: int = None 

761 

762 # ParamILS attributes 

763 self.__paramils_cpu_time_budget: int = None 

764 self.__paramils_min_runs: int = None 

765 self.__paramils_max_runs: int = None 

766 self.__paramils_random_restart: float = None 

767 self.__paramils_focused_approach: bool = None 

768 self.__paramils_use_cpu_time_in_tunertime: bool = None 

769 self.__paramils_cli_cores: int = None 

770 self.__paramils_max_iterations: int = None 

771 self.__paramils_number_initial_configurations: int = None 

772 

773 # Parallel portfolio attributes 

774 self.__parallel_portfolio_check_interval: int = None 

775 self.__parallel_portfolio_num_seeds_per_solver: int = None 

776 

777 # Slurm attributes 

778 self.__slurm_jobs_in_parallel: int = None 

779 self.__slurm_job_prepend: str = None 

780 

781 # The seed that has been used to set the random state 

782 self.random_state: Optional[int] = None 

783 

784 if file_path and file_path.exists(): 

785 self.read_settings_ini(file_path) 

786 

787 if argsv: 

788 self.apply_arguments(argsv) 

789 

790 def read_settings_ini(self: Settings, file_path: Path) -> None: 

791 """Read the settings from an INI file.""" 

792 if not file_path.exists(): 

793 raise ValueError(f"Settings file {file_path} does not exist.") 

794 # Read file 

795 file_settings = configparser.ConfigParser() 

796 file_settings.read(file_path) 

797 

798 # Set internal settings based on data read from FILE if they were read 

799 # successfully 

800 if file_settings.sections() == []: 

801 # Print error if unable to read the settings 

802 print( 

803 f"ERROR: Failed to read settings from {file_path} The file may " 

804 "have been empty or be in another format than INI." 

805 ) 

806 return 

807 

808 for section in file_settings.sections(): 

809 if section not in self.__settings.sections(): 

810 print(f'Unrecognised section: "{section}" in file {file_path} ignored') 

811 continue 

812 for option_name in file_settings.options(section): 

813 if option_name not in self.sections_options[section]: 

814 if section == Settings.SECTION_slurm: # Flexible section 

815 self.__settings.set( 

816 section, 

817 option_name, 

818 file_settings.get(section, option_name), 

819 ) 

820 else: 

821 print( 

822 f'Unrecognised section - option combination: "{section} ' 

823 f'{option_name}" in file {file_path} ignored' 

824 ) 

825 continue 

826 option_index = self.sections_options[section].index(option_name) 

827 option = self.sections_options[section][option_index] 

828 self.__settings.set( 

829 section, option.name, file_settings.get(section, option_name) 

830 ) 

831 del file_settings 

832 

833 def write_settings_ini(self: Settings, file_path: Path) -> None: 

834 """Write the settings to an INI file.""" 

835 # Create needed directories if they don't exist 

836 file_path.parent.mkdir(parents=True, exist_ok=True) 

837 # We don't write empty sections 

838 for s in self.__settings.sections(): 

839 if not self.__settings[s]: 

840 self.__settings.remove_section(s) 

841 with file_path.open("w") as fout: 

842 self.__settings.write(fout) 

843 for s in self.sections_options.keys(): 

844 if s not in self.__settings.sections(): 

845 self.__settings.add_section(s) 

846 

847 def write_used_settings(self: Settings) -> None: 

848 """Write the used settings to the default locations.""" 

849 # Write to latest settings file 

850 self.write_settings_ini(self.DEFAULT_previous_settings_path) 

851 

852 def apply_arguments(self: Settings, argsv: argparse.Namespace) -> None: 

853 """Apply the arguments to the settings.""" 

854 # Read a possible second file, that overwrites the first, where applicable 

855 # e.g. settings are not deleted, but overwritten where applicable 

856 if hasattr(argsv, "settings_file") and argsv.settings_file: 

857 self.read_settings_ini(argsv.settings_file) 

858 # Match all possible arguments to the settings 

859 for argument in argsv.__dict__.keys(): 

860 value = argsv.__dict__[argument] 

861 if value is None: 

862 continue # Skip None 

863 value = value.name if isinstance(value, Enum) else str(value) 

864 for section in self.sections_options.keys(): 

865 if argument in self.sections_options[section]: 

866 index = self.sections_options[section].index(argument) 

867 option = self.sections_options[section][index] 

868 self.__settings.set(option.section, option.name, value) 

869 break 

870 

871 def _abstract_getter(self: Settings, option: Option) -> Any: 

872 """Abstract getter method.""" 

873 if self.__settings.has_option(option.section, option.name): 

874 if option.type is bool: 

875 return self.__settings.getboolean(option.section, option.name) 

876 value = self.__settings.get(option.section, option.name) 

877 if not isinstance(value, option.type): 

878 if issubclass(option.type, Enum): 

879 return option.type[value.upper()] 

880 return option.type(value) # Attempt to resolve str to type 

881 return value 

882 return option.default_value 

883 

884 # General settings ### 

885 @property 

886 def objectives(self: Settings) -> list[SparkleObjective]: 

887 """Get the objectives for Sparkle.""" 

888 if self.__sparkle_objectives is None and self.__settings.has_option( 

889 Settings.SECTION_general, "objectives" 

890 ): 

891 objectives = self.__settings[Settings.SECTION_general]["objectives"] 

892 if "status" not in objectives: 

893 objectives += ",status:metric" 

894 if "cpu_time" not in objectives: 

895 objectives += ",cpu_time:metric" 

896 if "wall_time" not in objectives: 

897 objectives += ",wall_time:metric" 

898 if "memory" not in objectives: 

899 objectives += ",memory:metric" 

900 self.__sparkle_objectives = [ 

901 resolve_objective(obj) for obj in objectives.split(",") 

902 ] 

903 return self.__sparkle_objectives 

904 

905 @property 

906 def configurator(self: Settings) -> Configurator: 

907 """Get the configurator class (instance).""" 

908 if self.__general_sparkle_configurator is None and self.__settings.has_option( 

909 Settings.OPTION_configurator.section, Settings.OPTION_configurator.name 

910 ): 

911 self.__general_sparkle_configurator = cim.resolve_configurator( 

912 self.__settings.get( 

913 Settings.OPTION_configurator.section, 

914 Settings.OPTION_configurator.name, 

915 ) 

916 )() 

917 return self.__general_sparkle_configurator 

918 

919 @property 

920 def solver_cutoff_time(self: Settings) -> int: 

921 """Solver cutoff time in seconds.""" 

922 if self.__solver_cutoff_time is None: 

923 self.__solver_cutoff_time = self._abstract_getter( 

924 Settings.OPTION_solver_cutoff_time 

925 ) 

926 return self.__solver_cutoff_time 

927 

928 @property 

929 def extractor_cutoff_time(self: Settings) -> int: 

930 """Extractor cutoff time in seconds.""" 

931 if self.__extractor_cutoff_time is None: 

932 self.__extractor_cutoff_time = self._abstract_getter( 

933 Settings.OPTION_extractor_cutoff_time 

934 ) 

935 return self.__extractor_cutoff_time 

936 

937 @property 

938 def run_on(self: Settings) -> Runner: 

939 """On which compute to run (Local or Slurm).""" 

940 if self.__run_on is None: 

941 self.__run_on = self._abstract_getter(Settings.OPTION_run_on) 

942 return self.__run_on 

943 

944 @property 

945 def appendices(self: Settings) -> bool: 

946 """Whether to include appendices in the report.""" 

947 return self._abstract_getter(Settings.OPTION_appendices) 

948 

949 @property 

950 def verbosity_level(self: Settings) -> VerbosityLevel: 

951 """Verbosity level to use in CLI commands.""" 

952 if self.__verbosity_level is None: 

953 if self.__settings.has_option( 

954 Settings.OPTION_verbosity.section, Settings.OPTION_verbosity.name