Coverage for sparkle/configurator/configurator.py: 82%

1#!/usr/bin/env python3 

2# -*- coding: UTF-8 -*- 

3"""Configurator class to use different algorithm configurators like SMAC.""" 

4from __future__ import annotations 

5from pathlib import Path 

6 

7import runrunner as rrr 

8from runrunner import Runner, Run 

9 

10from sparkle.solver import Solver 

11from sparkle.instance import InstanceSet 

12from sparkle.structures import PerformanceDataFrame 

13from sparkle.types import SparkleObjective 

14 

15 

16class Configurator: 

17 """Abstact class to use different configurators like SMAC.""" 

18 configurator_cli_path = Path(__file__).parent.resolve() / "configurator_cli.py" 

19 

20 def __init__(self: Configurator, output_path: Path, 

21 base_dir: Path, tmp_path: Path, 

22 multi_objective_support: bool = False) -> None: 

23 """Initialize Configurator. 

24 

25 Args: 

26 output_path: Output directory of the Configurator. 

27 objectives: The list of Sparkle Objectives the configurator has to 

28 optimize. 

29 base_dir: Where to execute the configuration 

30 tmp_path: Path for the temporary files of the configurator, optional 

31 multi_objective_support: Whether the configurator supports 

32 multi objective optimization for solvers. 

33 """ 

34 self.output_path = output_path 

35 self.base_dir = base_dir 

36 self.tmp_path = tmp_path 

37 self.multiobjective = multi_objective_support 

38 self.scenario = None 

39 

40 @staticmethod 

41 def scenario_class() -> ConfigurationScenario: 

42 """Return the scenario class of the configurator.""" 

43 return ConfigurationScenario 

44 

45 def configure(self: Configurator, 

46 configuration_commands: list[str], 

47 data_target: PerformanceDataFrame, 

48 output: Path, 

49 scenario: ConfigurationScenario, 

50 validation_ids: list[int] = None, 

51 sbatch_options: list[str] = None, 

52 num_parallel_jobs: int = None, 

53 base_dir: Path = None, 

54 run_on: Runner = Runner.SLURM) -> Run: 

55 """Start configuration job. 

56 

57 Args: 

58 

59 scenario: ConfigurationScenario to execute. 

60 validate_after: Whether to validate the configuration on the training set 

61 afterwards or not. 

62 sbatch_options: List of slurm batch options to use 

63 num_parallel_jobs: The maximum number of jobs to run in parallel 

64 base_dir: The base_dir of RunRunner where the sbatch scripts will be placed 

65 run_on: On which platform to run the jobs. Default: Slurm. 

66 

67 Returns: 

68 A RunRunner Run object. 

69 """ 

70 runs = [rrr.add_to_queue( 

71 runner=run_on, 

72 cmd=configuration_commands, 

73 name=f"{self.name}: {scenario.solver.name} on {scenario.instance_set.name}", 

74 base_dir=base_dir, 

75 output_path=output, 

76 parallel_jobs=num_parallel_jobs, 

77 sbatch_options=sbatch_options)] 

78 

79 if validation_ids: 

80 validate = scenario.solver.run_performance_dataframe( 

81 scenario.instance_set, 

82 run_ids=validation_ids, 

83 performance_dataframe=data_target, 

84 cutoff_time=scenario.cutoff_time, 

85 sbatch_options=sbatch_options, 

86 log_dir=scenario.validation, 

87 base_dir=base_dir, 

88 dependencies=runs, 

89 job_name=f"{self.name}: Validating {len(validation_ids)} " 

90 f"{scenario.solver.name} Configurations on " 

91 f"{scenario.instance_set.name}", 

92 run_on=run_on, 

93 ) 

94 runs.append(validate) 

95 

96 if run_on == Runner.LOCAL: 

97 for run in runs: 

98 run.wait() 

99 return runs 

100 

101 @staticmethod 

102 def organise_output(output_source: Path, 

103 output_target: Path, 

104 scenario: ConfigurationScenario, 

105 run_id: int) -> None | str: 

106 """Method to restructure and clean up after a single configurator call. 

107 

108 Args: 

109 output_source: Path to the output file of the configurator run. 

110 output_target: Path to the Performance DataFrame to store result. 

111 scenario: ConfigurationScenario of the configuration. 

112 run_id: ID of the run of the configuration. 

113 """ 

114 raise NotImplementedError 

115 

116 def get_status_from_logs(self: Configurator) -> None: 

117 """Method to scan the log files of the configurator for warnings.""" 

118 raise NotImplementedError 

119 

120 

121class ConfigurationScenario: 

122 """Template class to handle a configuration scenarios.""" 

123 def __init__(self: ConfigurationScenario, 

124 solver: Solver, 

125 instance_set: InstanceSet, 

126 sparkle_objectives: list[SparkleObjective], 

127 parent_directory: Path) -> None: 

128 """Initialize scenario paths and names. 

129 

130 Args: 

131 solver: Solver that should be configured. 

132 instance_set: Instances object for the scenario. 

133 sparkle_objectives: Sparkle Objectives to optimize. 

134 parent_directory: Directory in which the scenario should be placed. 

135 """ 

136 self.solver = solver 

137 self.instance_set = instance_set 

138 self.sparkle_objectives = sparkle_objectives 

139 self.name = f"{self.solver.name}_{self.instance_set.name}" 

140 

141 if self.instance_set.size == 0: 

142 raise Exception("Cannot configure on an empty instance set " 

143 f"('{instance_set.name}').") 

144 

145 self.directory = parent_directory / self.name 

146 self.scenario_file_path = self.directory / f"{self.name}_scenario.txt" 

147 self.validation: Path = self.directory / "validation" 

148 self.tmp: Path = self.directory / "tmp" 

149 self.results_directory: Path = self.directory / "results" 

150 

151 def create_scenario(self: ConfigurationScenario, parent_directory: Path) -> None: 

152 """Create scenario with solver and instances in the parent directory. 

153 

154 This prepares all the necessary subdirectories related to configuration. 

155 

156 Args: 

157 parent_directory: Directory in which the scenario should be created. 

158 """ 

159 raise NotImplementedError 

160 

161 def create_scenario_file(self: ConfigurationScenario) -> Path: 

162 """Create a file with the configuration scenario. 

163 

164 Writes supplementary information to the target algorithm (algo =) as: 

165 algo = {configurator_target} {solver_directory} {sparkle_objective} 

166 """ 

167 raise NotImplementedError 

168 

169 def serialize(self: ConfigurationScenario) -> dict: 

170 """Serialize the configuration scenario.""" 

171 raise NotImplementedError 

172 

173 @classmethod 

174 def find_scenario(cls: ConfigurationScenario, 

175 directory: Path, 

176 solver: Solver, 

177 instance_set: InstanceSet) -> ConfigurationScenario: 

178 """Resolve a scenario from a directory and Solver / Training set.""" 

179 scenario_name = f"{solver.name}_{instance_set.name}" 

180 path = directory / f"{scenario_name}" / f"{scenario_name}_scenario.txt" 

181 if not path.exists(): 

182 return None 

183 return cls.from_file(path) 

184 

185 @staticmethod 

186 def from_file(scenario_file: Path) -> ConfigurationScenario: 

187 """Reads scenario file and initalises ConfigurationScenario.""" 

188 raise NotImplementedError