Coverage for src/sparkle/CLI/configure_solver.py: 66%

1#!/usr/bin/env python3 

2"""Sparkle command to configure a solver.""" 

3 

4from __future__ import annotations 

5from pathlib import Path 

6import argparse 

7import sys 

8 

9from runrunner import Runner 

10 

11from sparkle.CLI.help import global_variables as gv 

12from sparkle.CLI.help import logging as sl 

13from sparkle.CLI.initialise import check_for_initialise 

14from sparkle.CLI.help.nicknames import resolve_object_name, resolve_instance_name 

15from sparkle.CLI.help import argparse_custom as ac 

16 

17from sparkle.platform.settings_objects import Settings 

18from sparkle.structures import PerformanceDataFrame, FeatureDataFrame 

19from sparkle.solver import Solver 

20from sparkle.instance import Instance_Set 

21 

22 

23def parser_function() -> argparse.ArgumentParser: 

24 """Define the command line arguments.""" 

25 parser = argparse.ArgumentParser( 

26 description="Configure a solver in the platform.", 

27 epilog=( 

28 "Note that the test instance set is only used if the ``--ablation``" 

29 " or ``--validation`` flags are given" 

30 ), 

31 ) 

32 parser.add_argument(*ac.SolverArgument.names, **ac.SolverArgument.kwargs) 

33 parser.add_argument( 

34 *ac.InstanceSetTrainArgument.names, **ac.InstanceSetTrainArgument.kwargs 

35 ) 

36 parser.add_argument( 

37 *ac.InstanceSetTestArgument.names, **ac.InstanceSetTestArgument.kwargs 

38 ) 

39 parser.add_argument( 

40 *ac.TestSetRunAllConfigurationArgument.names, 

41 **ac.TestSetRunAllConfigurationArgument.kwargs, 

42 ) 

43 parser.add_argument(*ac.UseFeaturesArgument.names, **ac.UseFeaturesArgument.kwargs) 

44 # Settings Arguments 

45 parser.add_argument(*ac.SettingsFileArgument.names, **ac.SettingsFileArgument.kwargs) 

46 parser.add_argument( 

47 *Settings.OPTION_configurator.args, **Settings.OPTION_configurator.kwargs 

48 ) 

49 parser.add_argument( 

50 *Settings.OPTION_objectives.args, **Settings.OPTION_objectives.kwargs 

51 ) 

52 parser.add_argument( 

53 *Settings.OPTION_solver_cutoff_time.args, 

54 **Settings.OPTION_solver_cutoff_time.kwargs, 

55 ) 

56 parser.add_argument( 

57 *Settings.OPTION_configurator_solver_call_budget.args, 

58 **Settings.OPTION_configurator_solver_call_budget.kwargs, 

59 ) 

60 parser.add_argument( 

61 *Settings.OPTION_configurator_number_of_runs.args, 

62 **Settings.OPTION_configurator_number_of_runs.kwargs, 

63 ) 

64 parser.add_argument(*Settings.OPTION_run_on.args, **Settings.OPTION_run_on.kwargs) 

65 return parser 

66 

67 

68def main(argv: list[str]) -> None: 

69 """Main function of the configure solver command.""" 

70 parser = parser_function() 

71 

72 # Process command line arguments 

73 args = parser.parse_args(argv) 

74 settings = gv.settings(argsv=args) 

75 

76 # Log command call 

77 sl.log_command(sys.argv, settings.random_state) 

78 check_for_initialise() 

79 

80 configurator = settings.configurator 

81 # Check configurator is available 

82 if not configurator.check_requirements(verbose=True): 

83 print( 

84 f"{configurator.name} is not available. " 

85 "Please inspect possible warnings above." 

86 ) 

87 print(f"Would you like to install {configurator.name}? (Y/n)") 

88 if input().lower().strip() == "y": 

89 configurator.download_requirements() 

90 else: 

91 sys.exit() 

92 if not configurator.check_requirements(verbose=True): 

93 raise RuntimeError(f"Failed to install {configurator.name}.") 

94 sys.exit(-1) 

95 

96 # Compare current settings to latest.ini 

97 prev_settings = Settings(Path(Settings.DEFAULT_previous_settings_path)) 

98 Settings.check_settings_changes(settings, prev_settings) 

99 

100 configurator = gv.settings().configurator 

101 

102 # Check configurator is available 

103 if not configurator.check_requirements(verbose=True): 

104 print( 

105 f"{configurator.name} is not available. " 

106 "Please inspect possible warnings above." 

107 ) 

108 print(f"Would you like to install {configurator.name}? (Y/n)") 

109 if input().lower().strip() == "y": 

110 configurator.download_requirements() 

111 else: 

112 sys.exit() 

113 if not configurator.check_requirements(verbose=True): 

114 raise RuntimeError(f"Failed to install {configurator.name}.") 

115 sys.exit(-1) 

116 

117 # Compare current settings to latest.ini 

118 prev_settings = Settings(Path(Settings.DEFAULT_previous_settings_path)) 

119 Settings.check_settings_changes(gv.settings(), prev_settings) 

120 

121 solver: Solver = resolve_object_name( 

122 args.solver, 

123 gv.file_storage_data_mapping[gv.solver_nickname_list_path], 

124 settings.DEFAULT_solver_dir, 

125 class_name=Solver, 

126 ) 

127 if solver is None: 

128 raise ValueError(f"Solver {args.solver} not found.") 

129 instance_set_train = resolve_object_name( 

130 args.instance_set_train, 

131 gv.file_storage_data_mapping[gv.instances_nickname_path], 

132 settings.DEFAULT_instance_dir, 

133 Instance_Set, 

134 ) 

135 if instance_set_train is None: 

136 raise ValueError(f"Instance set {args.instance_set_train} not found.") 

137 instance_set_test = args.instance_set_test 

138 if instance_set_test is not None: 

139 instance_set_test = resolve_object_name( 

140 args.instance_set_test, 

141 gv.file_storage_data_mapping[gv.instances_nickname_path], 

142 settings.DEFAULT_instance_dir, 

143 Instance_Set, 

144 ) 

145 use_features = args.use_features 

146 run_on = settings.run_on 

147 

148 configurator_settings = settings.get_configurator_settings(configurator.name) 

149 

150 sparkle_objectives = settings.objectives 

151 if len(sparkle_objectives) > 1: 

152 print( 

153 f"WARNING: {configurator.name} does not have multi objective support. " 

154 f"Only the first objective ({sparkle_objectives[0]}) will be optimised." 

155 ) 

156 

157 performance_data = PerformanceDataFrame(settings.DEFAULT_performance_data_path) 

158 

159 # Check if given objectives are in the data frame 

160 for objective in sparkle_objectives: 

161 if objective.name not in performance_data.objective_names: 

162 print( 

163 f"WARNING: Objective {objective.name} not found in performance data. " 

164 "Adding to data frame." 

165 ) 

166 performance_data.add_objective(objective.name) 

167 

168 if use_features: 

169 feature_data = FeatureDataFrame(settings.DEFAULT_feature_data_path) 

170 # Check that the train instance set is in the feature data frame 

171 invalid = False 

172 remaining_instance_jobs = set( 

173 [instance for instance, _, _ in feature_data.remaining_jobs()] 

174 ) 

175 for instance_pair in instance_set_train.instance_pairs: 

176 if instance_pair not in feature_data.instance_pairs: 

177 print( 

178 f"ERROR: Train Instance {instance_pair} not found in feature data." 

179 ) 

180 invalid = True 

181 elif instance_pair in remaining_instance_jobs: # Check jobs 

182 print( 

183 f"ERROR: Features have not been computed for instance {instance_pair}." 

184 ) 

185 invalid = True 

186 if invalid: 

187 sys.exit(-1) 

188 configurator_settings.update({"feature_data": feature_data}) 

189 

190 number_of_runs = settings.configurator_number_of_runs 

191 output_path = settings.get_configurator_output_path(configurator) 

192 config_scenario = configurator.scenario_class()( 

193 solver, 

194 instance_set_train, 

195 sparkle_objectives, 

196 number_of_runs, 

197 output_path, 

198 **configurator_settings, 

199 ) 

200 

201 # Run the default configuration 

202 default_jobs = [ 

203 (solver, config_id, instance, run_id) 

204 for solver, config_id, instance, run_id in performance_data.remaining_jobs() 

205 if config_id == PerformanceDataFrame.default_configuration 

206 ] 

207 

208 sbatch_options = settings.sbatch_settings 

209 slurm_prepend = settings.slurm_job_prepend 

210 dependency_job_list = configurator.configure( 

211 scenario=config_scenario, 

212 data_target=performance_data, 

213 sbatch_options=sbatch_options, 

214 slurm_prepend=slurm_prepend, 

215 num_parallel_jobs=settings.slurm_jobs_in_parallel, 

216 base_dir=sl.caller_log_dir, 

217 run_on=run_on, 

218 ) 

219 

220 # If we have default configurations that need to be run, schedule them too 

221 if default_jobs: 

222 # Edit jobs to incorporate file paths. A set deduplicates instances that 

223 # repeat across default jobs (one job per remaining run), since 

224 # run_performance_dataframe re-expands over all runs itself. 

225 # NOTE: This relies on resolve_instance_name returning hashable values (a Path, 

226 # or a space-joined str for multi-file instances). If it is ever changed to 

227 # return a list[Path], set.add would raise TypeError: unhashable type: 'list', 

228 # and this would need to join/normalise the paths first. 

229 # instances = list(dict.fromkeys(...)) 

230 instances = set() 

231 for _, _, (instance_set, instance_name), _ in default_jobs: 

232 instance_path = resolve_instance_name( 

233 instance_set, instance_name, settings.DEFAULT_instance_dir 

234 ) 

235 instances.add(instance_path) 

236 default_job = solver.run_performance_dataframe( 

237 instances, 

238 performance_data, 

239 PerformanceDataFrame.default_configuration, 

240 sbatch_options=sbatch_options, 

241 slurm_prepend=slurm_prepend, 

242 cutoff_time=config_scenario.solver_cutoff_time, 

243 log_dir=config_scenario.validation, 

244 base_dir=sl.caller_log_dir, 

245 job_name=f"Default Configuration: {solver.name} Validation on " 

246 f"{instance_set_train.name}", 

247 run_on=run_on, 

248 ) 

249 dependency_job_list.append(default_job) 

250 

251 if instance_set_test is not None: 

252 # Schedule test set jobs 

253 if args.test_set_run_all_configurations: 

254 # TODO: Schedule test set runs for all configurations 

255 print("Running all configurations on test set is not implemented yet.") 

256 pass 

257 else: 

258 # We place the results in the index we just added 

259 run_index = list( 

260 set( 

261 [ 

262 performance_data.get_instance_num_runs( 

263 instance_set_test.name, test_instance_name 

264 ) 

265 for test_instance_name in instance_set_test.instance_names 

266 ] 

267 ) 

268 ) 

269 test_set_job = solver.run_performance_dataframe( 

270 instance_set_test, 

271 performance_data, 

272 run_ids=run_index, 

273 cutoff_time=config_scenario.solver_cutoff_time, 

274 objective=config_scenario.sparkle_objective, 

275 train_set=instance_set_train, 

276 sbatch_options=sbatch_options, 

277 slurm_prepend=slurm_prepend, 

278 log_dir=config_scenario.validation, 

279 base_dir=sl.caller_log_dir, 

280 dependencies=dependency_job_list, 

281 job_name=f"Best Configuration: {solver.name} Validation on " 

282 f"{instance_set_test.name}", 

283 run_on=run_on, 

284 ) 

285 dependency_job_list.append(test_set_job) 

286 

287 if run_on == Runner.SLURM: 

288 job_id_str = ",".join([run.run_id for run in dependency_job_list]) 

289 print( 

290 f"Running {configurator.name} configuration through Slurm with job " 

291 f"id(s): {job_id_str}" 

292 ) 

293 else: 

294 print("Running configuration finished!") 

295 

296 # Write used settings to file 

297 settings.write_used_settings() 

298 # Write used scenario to file 

299 sys.exit(0) 

300 

301 

302if __name__ == "__main__": 

303 main(sys.argv[1:])