Coverage for sparkle/CLI/configure_solver.py: 66%
122 statements
« prev ^ index » next coverage.py v7.10.7, created at 2025-09-29 10:17 +0000
1#!/usr/bin/env python3
2"""Sparkle command to configure a solver."""
4from __future__ import annotations
5from pathlib import Path
6import argparse
7import sys
9from runrunner import Runner
11from sparkle.CLI.help import global_variables as gv
12from sparkle.CLI.help import logging as sl
13from sparkle.CLI.initialise import check_for_initialise
14from sparkle.CLI.help.nicknames import resolve_object_name, resolve_instance_name
15from sparkle.CLI.help import argparse_custom as ac
17from sparkle.platform.settings_objects import Settings
18from sparkle.structures import PerformanceDataFrame, FeatureDataFrame
19from sparkle.solver import Solver
20from sparkle.instance import Instance_Set
23def parser_function() -> argparse.ArgumentParser:
24 """Define the command line arguments."""
25 parser = argparse.ArgumentParser(
26 description="Configure a solver in the platform.",
27 epilog=(
28 "Note that the test instance set is only used if the ``--ablation``"
29 " or ``--validation`` flags are given"
30 ),
31 )
32 parser.add_argument(*ac.SolverArgument.names, **ac.SolverArgument.kwargs)
33 parser.add_argument(
34 *ac.InstanceSetTrainArgument.names, **ac.InstanceSetTrainArgument.kwargs
35 )
36 parser.add_argument(
37 *ac.InstanceSetTestArgument.names, **ac.InstanceSetTestArgument.kwargs
38 )
39 parser.add_argument(
40 *ac.TestSetRunAllConfigurationArgument.names,
41 **ac.TestSetRunAllConfigurationArgument.kwargs,
42 )
43 parser.add_argument(*ac.UseFeaturesArgument.names, **ac.UseFeaturesArgument.kwargs)
44 # Settings Arguments
45 parser.add_argument(*ac.SettingsFileArgument.names, **ac.SettingsFileArgument.kwargs)
46 parser.add_argument(
47 *Settings.OPTION_configurator.args, **Settings.OPTION_configurator.kwargs
48 )
49 parser.add_argument(
50 *Settings.OPTION_objectives.args, **Settings.OPTION_objectives.kwargs
51 )
52 parser.add_argument(
53 *Settings.OPTION_solver_cutoff_time.args,
54 **Settings.OPTION_solver_cutoff_time.kwargs,
55 )
56 parser.add_argument(
57 *Settings.OPTION_configurator_solver_call_budget.args,
58 **Settings.OPTION_configurator_solver_call_budget.kwargs,
59 )
60 parser.add_argument(
61 *Settings.OPTION_configurator_number_of_runs.args,
62 **Settings.OPTION_configurator_number_of_runs.kwargs,
63 )
64 parser.add_argument(*Settings.OPTION_run_on.args, **Settings.OPTION_run_on.kwargs)
65 return parser
68def main(argv: list[str]) -> None:
69 """Main function of the configure solver command."""
70 parser = parser_function()
72 # Process command line arguments
73 args = parser.parse_args(argv)
74 settings = gv.settings(argsv=args)
76 # Log command call
77 sl.log_command(sys.argv, settings.random_state)
78 check_for_initialise()
80 configurator = settings.configurator
81 # Check configurator is available
82 if not configurator.check_requirements(verbose=True):
83 print(
84 f"{configurator.name} is not available. "
85 "Please inspect possible warnings above."
86 )
87 print(f"Would you like to install {configurator.name}? (Y/n)")
88 if input().lower().strip() == "y":
89 configurator.download_requirements()
90 else:
91 sys.exit()
92 if not configurator.check_requirements(verbose=True):
93 raise RuntimeError(f"Failed to install {configurator.name}.")
94 sys.exit(-1)
96 # Compare current settings to latest.ini
97 prev_settings = Settings(Path(Settings.DEFAULT_previous_settings_path))
98 Settings.check_settings_changes(settings, prev_settings)
100 configurator = gv.settings().configurator
102 # Check configurator is available
103 if not configurator.check_requirements(verbose=True):
104 print(
105 f"{configurator.name} is not available. "
106 "Please inspect possible warnings above."
107 )
108 print(f"Would you like to install {configurator.name}? (Y/n)")
109 if input().lower().strip() == "y":
110 configurator.download_requirements()
111 else:
112 sys.exit()
113 if not configurator.check_requirements(verbose=True):
114 raise RuntimeError(f"Failed to install {configurator.name}.")
115 sys.exit(-1)
117 # Compare current settings to latest.ini
118 prev_settings = Settings(Path(Settings.DEFAULT_previous_settings_path))
119 Settings.check_settings_changes(gv.settings(), prev_settings)
121 solver: Solver = resolve_object_name(
122 args.solver,
123 gv.file_storage_data_mapping[gv.solver_nickname_list_path],
124 settings.DEFAULT_solver_dir,
125 class_name=Solver,
126 )
127 if solver is None:
128 raise ValueError(f"Solver {args.solver} not found.")
129 instance_set_train = resolve_object_name(
130 args.instance_set_train,
131 gv.file_storage_data_mapping[gv.instances_nickname_path],
132 settings.DEFAULT_instance_dir,
133 Instance_Set,
134 )
135 if instance_set_train is None:
136 raise ValueError(f"Instance set {args.instance_set_train} not found.")
137 instance_set_test = args.instance_set_test
138 if instance_set_test is not None:
139 instance_set_test = resolve_object_name(
140 args.instance_set_test,
141 gv.file_storage_data_mapping[gv.instances_nickname_path],
142 settings.DEFAULT_instance_dir,
143 Instance_Set,
144 )
145 use_features = args.use_features
146 run_on = settings.run_on
148 configurator_settings = settings.get_configurator_settings(configurator.name)
150 sparkle_objectives = settings.objectives
151 if len(sparkle_objectives) > 1:
152 print(
153 f"WARNING: {configurator.name} does not have multi objective support. "
154 f"Only the first objective ({sparkle_objectives[0]}) will be optimised."
155 )
157 performance_data = PerformanceDataFrame(settings.DEFAULT_performance_data_path)
159 # Check if given objectives are in the data frame
160 for objective in sparkle_objectives:
161 if objective.name not in performance_data.objective_names:
162 print(
163 f"WARNING: Objective {objective.name} not found in performance data. "
164 "Adding to data frame."
165 )
166 performance_data.add_objective(objective.name)
168 if use_features:
169 feature_data = FeatureDataFrame(settings.DEFAULT_feature_data_path)
170 # Check that the train instance set is in the feature data frame
171 invalid = False
172 remaining_instance_jobs = set(
173 [instance for instance, _, _ in feature_data.remaining_jobs()]
174 )
175 for instance in instance_set_train.instance_paths:
176 if str(instance) not in feature_data.instances:
177 print(f"ERROR: Train Instance {instance} not found in feature data.")
178 invalid = True
179 elif instance in remaining_instance_jobs: # Check jobs
180 print(f"ERROR: Features have not been computed for instance {instance}.")
181 invalid = True
182 if invalid:
183 sys.exit(-1)
184 configurator_settings.update({"feature_data": feature_data})
186 number_of_runs = settings.configurator_number_of_runs
187 output_path = settings.get_configurator_output_path(configurator)
188 config_scenario = configurator.scenario_class()(
189 solver,
190 instance_set_train,
191 sparkle_objectives,
192 number_of_runs,
193 output_path,
194 **configurator_settings,
195 )
197 # Run the default configuration
198 default_jobs = [
199 (solver, config_id, instance, run_id)
200 for solver, config_id, instance, run_id in performance_data.get_job_list()
201 if config_id == PerformanceDataFrame.default_configuration
202 ]
204 sbatch_options = settings.sbatch_settings
205 slurm_prepend = settings.slurm_job_prepend
206 dependency_job_list = configurator.configure(
207 scenario=config_scenario,
208 data_target=performance_data,
209 sbatch_options=sbatch_options,
210 slurm_prepend=slurm_prepend,
211 num_parallel_jobs=settings.slurm_jobs_in_parallel,
212 base_dir=sl.caller_log_dir,
213 run_on=run_on,
214 )
216 # If we have default configurations that need to be run, schedule them too
217 if default_jobs:
218 # Edit jobs to incorporate file paths
219 instances = []
220 for _, _, instance, _ in default_jobs:
221 instance_path = resolve_instance_name(
222 instance, settings.DEFAULT_instance_dir
223 )
224 instances.append(instance_path)
225 default_job = solver.run_performance_dataframe(
226 instances,
227 performance_data,
228 PerformanceDataFrame.default_configuration,
229 sbatch_options=sbatch_options,
230 slurm_prepend=slurm_prepend,
231 cutoff_time=config_scenario.solver_cutoff_time,
232 log_dir=config_scenario.validation,
233 base_dir=sl.caller_log_dir,
234 job_name=f"Default Configuration: {solver.name} Validation on "
235 f"{instance_set_train.name}",
236 run_on=run_on,
237 )
238 dependency_job_list.append(default_job)
240 if instance_set_test is not None:
241 # Schedule test set jobs
242 if args.test_set_run_all_configurations:
243 # TODO: Schedule test set runs for all configurations
244 print("Running all configurations on test set is not implemented yet.")
245 pass
246 else:
247 # We place the results in the index we just added
248 run_index = list(
249 set(
250 [
251 performance_data.get_instance_num_runs(str(i))
252 for i in instance_set_test.instance_names
253 ]
254 )
255 )
256 test_set_job = solver.run_performance_dataframe(
257 instance_set_test,
258 performance_data,
259 run_ids=run_index,
260 cutoff_time=config_scenario.solver_cutoff_time,
261 objective=config_scenario.sparkle_objective,
262 train_set=instance_set_train,
263 sbatch_options=sbatch_options,
264 slurm_prepend=slurm_prepend,
265 log_dir=config_scenario.validation,
266 base_dir=sl.caller_log_dir,
267 dependencies=dependency_job_list,
268 job_name=f"Best Configuration: {solver.name} Validation on "
269 f"{instance_set_test.name}",
270 run_on=run_on,
271 )
272 dependency_job_list.append(test_set_job)
274 if run_on == Runner.SLURM:
275 job_id_str = ",".join([run.run_id for run in dependency_job_list])
276 print(
277 f"Running {configurator.name} configuration through Slurm with job "
278 f"id(s): {job_id_str}"
279 )
280 else:
281 print("Running configuration finished!")
283 # Write used settings to file
284 settings.write_used_settings()
285 # Write used scenario to file
286 sys.exit(0)
289if __name__ == "__main__":
290 main(sys.argv[1:])