Coverage for sparkle/CLI/configure_solver.py: 70%
125 statements
« prev ^ index » next coverage.py v7.9.1, created at 2025-07-01 13:21 +0000
1#!/usr/bin/env python3
2"""Sparkle command to configure a solver."""
3from __future__ import annotations
4from pathlib import Path
5import argparse
6import sys
8from runrunner import Runner
10from sparkle.CLI.help import global_variables as gv
11from sparkle.CLI.help import logging as sl
12from sparkle.CLI.initialise import check_for_initialise
13from sparkle.CLI.help.nicknames import resolve_object_name, resolve_instance_name
14from sparkle.CLI.help import argparse_custom as ac
16from sparkle.platform.settings_objects import Settings, SettingState
17from sparkle.structures import PerformanceDataFrame, FeatureDataFrame
18from sparkle.solver import Solver
19from sparkle.instance import Instance_Set
22def parser_function() -> argparse.ArgumentParser:
23 """Define the command line arguments."""
24 parser = argparse.ArgumentParser(
25 description="Configure a solver in the platform.",
26 epilog=("Note that the test instance set is only used if the ``--ablation``"
27 " or ``--validation`` flags are given"))
28 parser.add_argument(*ac.ConfiguratorArgument.names,
29 **ac.ConfiguratorArgument.kwargs)
30 parser.add_argument(*ac.SolverArgument.names,
31 **ac.SolverArgument.kwargs)
32 parser.add_argument(*ac.InstanceSetTrainArgument.names,
33 **ac.InstanceSetTrainArgument.kwargs)
34 parser.add_argument(*ac.InstanceSetTestArgument.names,
35 **ac.InstanceSetTestArgument.kwargs)
36 parser.add_argument(*ac.TestSetRunAllConfigurationArgument.names,
37 **ac.TestSetRunAllConfigurationArgument.kwargs)
38 parser.add_argument(*ac.ObjectivesArgument.names,
39 **ac.ObjectivesArgument.kwargs)
40 parser.add_argument(*ac.SolverCutOffTimeArgument.names,
41 **ac.SolverCutOffTimeArgument.kwargs)
42 parser.add_argument(*ac.SolverCallsArgument.names,
43 **ac.SolverCallsArgument.kwargs)
44 parser.add_argument(*ac.NumberOfRunsConfigurationArgument.names,
45 **ac.NumberOfRunsConfigurationArgument.kwargs)
46 parser.add_argument(*ac.SettingsFileArgument.names,
47 **ac.SettingsFileArgument.kwargs)
48 parser.add_argument(*ac.UseFeaturesArgument.names,
49 **ac.UseFeaturesArgument.kwargs)
50 parser.add_argument(*ac.RunOnArgument.names,
51 **ac.RunOnArgument.kwargs)
52 return parser
55def apply_settings_from_args(args: argparse.Namespace) -> None:
56 """Apply command line arguments to settings.
58 Args:
59 args: Arguments object created by ArgumentParser.
60 """
61 if args.settings_file is not None:
62 gv.settings().read_settings_ini(args.settings_file, SettingState.CMD_LINE)
63 if args.configurator is not None:
64 gv.settings().set_general_sparkle_configurator(
65 args.configurator, SettingState.CMD_LINE)
66 if args.objectives is not None:
67 gv.settings().set_general_sparkle_objectives(
68 args.objectives, SettingState.CMD_LINE)
69 if args.solver_cutoff_time is not None:
70 gv.settings().set_general_solver_cutoff_time(
71 args.solver_cutoff_time, SettingState.CMD_LINE)
72 if args.solver_calls is not None:
73 gv.settings().set_configurator_solver_calls(
74 args.solver_calls, SettingState.CMD_LINE)
75 if args.number_of_runs is not None:
76 gv.settings().set_configurator_number_of_runs(
77 args.number_of_runs, SettingState.CMD_LINE)
78 if args.run_on is not None:
79 gv.settings().set_run_on(
80 args.run_on.value, SettingState.CMD_LINE)
83def main(argv: list[str]) -> None:
84 """Main function of the configure solver command."""
85 # Log command call
86 sl.log_command(sys.argv)
87 check_for_initialise()
89 parser = parser_function()
91 # Process command line arguments
92 args = parser.parse_args(argv)
93 apply_settings_from_args(args)
95 configurator = gv.settings().get_general_sparkle_configurator()
97 # Check configurator is available
98 if not configurator.check_requirements(verbose=True):
99 print(f"{configurator.name} is not available. "
100 "Please inspect possible warnings above.")
101 print(f"Would you like to install {configurator.name}? (Y/n)")
102 if input().lower().strip() == "y":
103 configurator.download_requirements()
104 else:
105 sys.exit()
106 if not configurator.check_requirements(verbose=True):
107 raise RuntimeError(f"Failed to install {configurator.name}.")
108 sys.exit(-1)
110 # Compare current settings to latest.ini
111 prev_settings = Settings(Path(Settings.DEFAULT_previous_settings_path))
112 Settings.check_settings_changes(gv.settings(), prev_settings)
114 solver: Solver = resolve_object_name(
115 args.solver,
116 gv.file_storage_data_mapping[gv.solver_nickname_list_path],
117 gv.settings().DEFAULT_solver_dir, class_name=Solver)
118 if solver is None:
119 raise ValueError(f"Solver {args.solver} not found.")
120 instance_set_train = resolve_object_name(
121 args.instance_set_train,
122 gv.file_storage_data_mapping[gv.instances_nickname_path],
123 gv.settings().DEFAULT_instance_dir, Instance_Set)
124 if instance_set_train is None:
125 raise ValueError(f"Instance set {args.instance_set_train} not found.")
126 instance_set_test = args.instance_set_test
127 if instance_set_test is not None:
128 instance_set_test = resolve_object_name(
129 args.instance_set_test,
130 gv.file_storage_data_mapping[gv.instances_nickname_path],
131 gv.settings().DEFAULT_instance_dir, Instance_Set)
132 use_features = args.use_features
133 run_on = gv.settings().get_run_on()
135 configurator_settings = gv.settings().get_configurator_settings(configurator.name)
137 sparkle_objectives =\
138 gv.settings().get_general_sparkle_objectives()
139 if len(sparkle_objectives) > 1:
140 print(f"WARNING: {configurator.name} does not have multi objective support. "
141 f"Only the first objective ({sparkle_objectives[0]}) will be optimised.")
143 performance_data = PerformanceDataFrame(gv.settings().DEFAULT_performance_data_path)
145 # Check if given objectives are in the data frame
146 for objective in sparkle_objectives:
147 if objective.name not in performance_data.objective_names:
148 print(f"WARNING: Objective {objective.name} not found in performance data. "
149 "Adding to data frame.")
150 performance_data.add_objective(objective.name)
152 if use_features:
153 feature_data = FeatureDataFrame(gv.settings().DEFAULT_feature_data_path)
154 # Check that the train instance set is in the feature data frame
155 invalid = False
156 remaining_instance_jobs =\
157 set([instance for instance, _, _ in feature_data.remaining_jobs()])
158 for instance in instance_set_train.instance_paths:
159 if str(instance) not in feature_data.instances:
160 print(f"ERROR: Train Instance {instance} not found in feature data.")
161 invalid = True
162 elif instance in remaining_instance_jobs: # Check jobs
163 print(f"ERROR: Features have not been computed for instance {instance}.")
164 invalid = True
165 if invalid:
166 sys.exit(-1)
167 configurator_settings.update({"feature_data": feature_data})
169 number_of_runs = gv.settings().get_configurator_number_of_runs()
170 output_path = gv.settings().get_configurator_output_path(configurator)
171 config_scenario = configurator.scenario_class()(
172 solver, instance_set_train, sparkle_objectives, number_of_runs,
173 output_path, **configurator_settings)
175 # Run the default configuration
176 default_jobs = [(solver, config_id, instance, run_id)
177 for solver, config_id, instance, run_id
178 in performance_data.get_job_list()
179 if config_id == PerformanceDataFrame.default_configuration]
181 sbatch_options = gv.settings().get_slurm_extra_options(as_args=True)
182 slurm_prepend = gv.settings().get_slurm_job_prepend()
183 dependency_job_list = configurator.configure(
184 scenario=config_scenario,
185 data_target=performance_data,
186 sbatch_options=sbatch_options,
187 slurm_prepend=slurm_prepend,
188 num_parallel_jobs=gv.settings().get_number_of_jobs_in_parallel(),
189 base_dir=sl.caller_log_dir,
190 run_on=run_on)
192 # If we have default configurations that need to be run, schedule them too
193 if default_jobs:
194 # Edit jobs to incorporate file paths
195 instances = []
196 for _, _, instance, _ in default_jobs:
197 instance_path = resolve_instance_name(
198 instance, gv.settings().DEFAULT_instance_dir)
199 instances.append(instance_path)
200 default_job = solver.run_performance_dataframe(
201 instances, PerformanceDataFrame.default_configuration,
202 performance_data,
203 sbatch_options=sbatch_options,
204 slurm_prepend=slurm_prepend,
205 cutoff_time=config_scenario.solver_cutoff_time,
206 log_dir=config_scenario.validation,
207 base_dir=sl.caller_log_dir,
208 job_name=f"Default Configuration: {solver.name} Validation on "
209 f"{instance_set_train.name}",
210 run_on=run_on)
211 dependency_job_list.append(default_job)
213 if instance_set_test is not None:
214 # Schedule test set jobs
215 if args.test_set_run_all_configurations:
216 # TODO: Schedule test set runs for all configurations
217 print("Running all configurations on test set is not implemented yet.")
218 pass
219 else:
220 # We place the results in the index we just added
221 run_index = list(set([performance_data.get_instance_num_runs(str(i))
222 for i in instance_set_test.instance_names]))
223 test_set_job = solver.run_performance_dataframe(
224 instance_set_test,
225 run_index,
226 performance_data,
227 cutoff_time=config_scenario.solver_cutoff_time,
228 objective=config_scenario.sparkle_objective,
229 train_set=instance_set_train,
230 sbatch_options=sbatch_options,
231 slurm_prepend=slurm_prepend,
232 log_dir=config_scenario.validation,
233 base_dir=sl.caller_log_dir,
234 dependencies=dependency_job_list,
235 job_name=f"Best Configuration: {solver.name} Validation on "
236 f"{instance_set_test.name}",
237 run_on=run_on)
238 dependency_job_list.append(test_set_job)
240 if run_on == Runner.SLURM:
241 job_id_str = ",".join([run.run_id for run in dependency_job_list])
242 print(f"Running {configurator.name} configuration through Slurm with job "
243 f"id(s): {job_id_str}")
244 else:
245 print("Running configuration finished!")
247 # Write used settings to file
248 gv.settings().write_used_settings()
249 # Write used scenario to file
250 sys.exit(0)
253if __name__ == "__main__":
254 main(sys.argv[1:])