Coverage for sparkle/CLI/configure_solver.py: 0%
131 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-09-27 09:10 +0000
1#!/usr/bin/env python3
2"""Sparkle command to configure a solver."""
3from __future__ import annotations
5import argparse
6import sys
7import os
8from pathlib import Path
9from pandas import DataFrame
11from runrunner.base import Runner, Run
12import runrunner as rrr
14from sparkle.CLI.help import global_variables as gv
15from sparkle.CLI.help import logging as sl
16from sparkle.platform.settings_objects import SettingState
17from sparkle.CLI.help.reporting_scenario import Scenario
18from sparkle.structures import FeatureDataFrame
19from sparkle.platform import CommandName, COMMAND_DEPENDENCIES
20from sparkle.configurator.configurator import Configurator
21from sparkle.CLI.help.nicknames import resolve_object_name
22from sparkle.solver import Solver
23from sparkle.CLI.initialise import check_for_initialise
24from sparkle.CLI.help import argparse_custom as ac
25from sparkle.instance import instance_set, InstanceSet
28def parser_function() -> argparse.ArgumentParser:
29 """Define the command line arguments."""
30 parser = argparse.ArgumentParser(
31 description="Configure a solver in the Sparkle platform.",
32 epilog=("Note that the test instance set is only used if the ``--ablation``"
33 " or ``--validation`` flags are given"))
34 parser.add_argument(*ac.ConfiguratorArgument.names,
35 **ac.ConfiguratorArgument.kwargs)
36 parser.add_argument(*ac.SolverArgument.names,
37 **ac.SolverArgument.kwargs)
38 parser.add_argument(*ac.InstanceSetTrainArgument.names,
39 **ac.InstanceSetTrainArgument.kwargs)
40 parser.add_argument(*ac.InstanceSetTestArgument.names,
41 **ac.InstanceSetTestArgument.kwargs)
42 parser.add_argument(*ac.SparkleObjectiveArgument.names,
43 **ac.SparkleObjectiveArgument.kwargs)
44 parser.add_argument(*ac.TargetCutOffTimeConfigurationArgument.names,
45 **ac.TargetCutOffTimeConfigurationArgument.kwargs)
46 parser.add_argument(*ac.WallClockTimeArgument.names,
47 **ac.WallClockTimeArgument.kwargs)
48 parser.add_argument(*ac.CPUTimeArgument.names,
49 **ac.CPUTimeArgument.kwargs)
50 parser.add_argument(*ac.SolverCallsArgument.names,
51 **ac.SolverCallsArgument.kwargs)
52 parser.add_argument(*ac.NumberOfRunsConfigurationArgument.names,
53 **ac.NumberOfRunsConfigurationArgument.kwargs)
54 parser.add_argument(*ac.SettingsFileArgument.names,
55 **ac.SettingsFileArgument.kwargs)
56 parser.add_argument(*ac.UseFeaturesArgument.names,
57 **ac.UseFeaturesArgument.kwargs)
58 parser.add_argument(*ac.ValidateArgument.names,
59 **ac.ValidateArgument.kwargs)
60 parser.add_argument(*ac.AblationArgument.names,
61 **ac.AblationArgument.kwargs)
62 parser.add_argument(*ac.RunOnArgument.names,
63 **ac.RunOnArgument.kwargs)
64 return parser
67def apply_settings_from_args(args: argparse.Namespace) -> None:
68 """Apply command line arguments to settings.
70 Args:
71 args: Arguments object created by ArgumentParser.
72 """
73 if args.settings_file is not None:
74 gv.settings().read_settings_ini(args.settings_file, SettingState.CMD_LINE)
75 if args.objectives is not None:
76 gv.settings().set_general_sparkle_objectives(
77 args.objectives, SettingState.CMD_LINE)
78 if args.target_cutoff_time is not None:
79 gv.settings().set_general_target_cutoff_time(
80 args.target_cutoff_time, SettingState.CMD_LINE)
81 if args.wallclock_time is not None:
82 gv.settings().set_config_wallclock_time(
83 args.wallclock_time, SettingState.CMD_LINE)
84 if args.cpu_time is not None:
85 gv.settings().set_config_cpu_time(
86 args.cpu_time, SettingState.CMD_LINE)
87 if args.solver_calls is not None:
88 gv.settings().set_config_solver_calls(
89 args.solver_calls, SettingState.CMD_LINE)
90 if args.number_of_runs is not None:
91 gv.settings().set_config_number_of_runs(
92 args.number_of_runs, SettingState.CMD_LINE)
93 if args.run_on is not None:
94 gv.settings().set_run_on(
95 args.run_on.value, SettingState.CMD_LINE)
98def run_after(solver: Path,
99 train_set: InstanceSet,
100 test_set: InstanceSet,
101 dependency: list[Run],
102 command: CommandName,
103 run_on: Runner = Runner.SLURM) -> Run:
104 """Add a command to run after configuration to RunRunner queue.
106 Args:
107 solver: Path (object) to solver.
108 train_set: Instances used for training.
109 test_set: Instances used for testing.
110 dependency: List of job dependencies.
111 command: The command to run. Currently supported: Validation and Ablation.
112 run_on: Whether the job is executed on Slurm or locally.
114 Returns:
115 RunRunner Run object regarding the callback
116 """
117 cmd_file = "validate_configured_vs_default.py"
118 if command == CommandName.RUN_ABLATION:
119 cmd_file = "run_ablation.py"
121 command_line = f"./sparkle/CLI/{cmd_file} --settings-file Settings/latest.ini "\
122 f"--solver {solver.name} --instance-set-train {train_set.directory}"\
123 f" --run-on {run_on}"
124 if test_set is not None:
125 command_line += f" --instance-set-test {test_set.directory}"
127 run = rrr.add_to_queue(
128 runner=run_on,
129 cmd=command_line,
130 name=command,
131 dependencies=dependency,
132 base_dir=sl.caller_log_dir,
133 srun_options=["-N1", "-n1"],
134 sbatch_options=gv.settings().get_slurm_extra_options(as_args=True))
136 if run_on == Runner.LOCAL:
137 print("Waiting for the local calculations to finish.")
138 run.wait()
139 return run
142if __name__ == "__main__":
143 # Log command call
144 sl.log_command(sys.argv)
146 parser = parser_function()
148 # Process command line arguments
149 args = parser.parse_args()
151 apply_settings_from_args(args)
153 validate = args.validate
154 ablation = args.ablation
155 solver = resolve_object_name(
156 args.solver,
157 gv.file_storage_data_mapping[gv.solver_nickname_list_path],
158 gv.settings().DEFAULT_solver_dir, class_name=Solver)
159 instance_set_train = resolve_object_name(
160 args.instance_set_train,
161 gv.file_storage_data_mapping[gv.instances_nickname_path],
162 gv.settings().DEFAULT_instance_dir, instance_set)
163 instance_set_test = args.instance_set_test
164 if instance_set_test is not None:
165 instance_set_test = resolve_object_name(
166 args.instance_set_test,
167 gv.file_storage_data_mapping[gv.instances_nickname_path],
168 gv.settings().DEFAULT_instance_dir, instance_set)
169 use_features = args.use_features
170 run_on = gv.settings().get_run_on()
171 if args.configurator is not None:
172 gv.settings().set_general_sparkle_configurator(
173 value=getattr(Configurator, args.configurator),
174 origin=SettingState.CMD_LINE)
176 # Check if Solver and instance sets were resolved
177 check_for_initialise(COMMAND_DEPENDENCIES[CommandName.CONFIGURE_SOLVER])
179 feature_data_df = None
180 if use_features:
181 feature_data = FeatureDataFrame(gv.settings().DEFAULT_feature_data_path)
183 data_dict = {}
184 feature_data_df = feature_data.dataframe
186 for label, row in feature_data_df.iterrows():
187 # os.path.split(os.path.split(label)[0])[1] gives the dir/instance set name
188 if os.path.split(os.path.split(label)[0])[1] == instance_set_train.name:
189 if row.empty:
190 print("No feature data exists for the given training set, please "
191 "run add_feature_extractor.py, then compute_features.py")
192 sys.exit(-1)
194 new_label = (f"../../../instances/{instance_set_train.name}/"
195 + os.path.split(label)[1])
196 data_dict[new_label] = row
198 feature_data_df = DataFrame.from_dict(data_dict, orient="index",
199 columns=feature_data_df.columns)
201 if feature_data.has_missing_value():
202 print("You have unfinished feature computation jobs, please run "
203 "`sparkle compute features`")
204 sys.exit(-1)
206 for index, column in enumerate(feature_data_df):
207 feature_data_df.rename(columns={column: f"Feature{index+1}"}, inplace=True)
209 number_of_runs = gv.settings().get_config_number_of_runs()
210 solver_calls = gv.settings().get_config_solver_calls()
211 cpu_time = gv.settings().get_config_cpu_time()
212 wallclock_time = gv.settings().get_config_wallclock_time()
213 cutoff_time = gv.settings().get_general_target_cutoff_time()
214 cutoff_length = gv.settings().get_configurator_target_cutoff_length()
215 sparkle_objectives =\
216 gv.settings().get_general_sparkle_objectives()
217 configurator = gv.settings().get_general_sparkle_configurator()
218 config_scenario = configurator.scenario_class(
219 solver, instance_set_train, number_of_runs, solver_calls, cpu_time,
220 wallclock_time, cutoff_time, cutoff_length, sparkle_objectives, use_features,
221 configurator.configurator_target, feature_data_df)
223 sbatch_options = gv.settings().get_slurm_extra_options(as_args=True)
224 dependency_job_list = configurator.configure(
225 scenario=config_scenario,
226 sbatch_options=sbatch_options,
227 num_parallel_jobs=gv.settings().get_number_of_jobs_in_parallel(),
228 base_dir=sl.caller_log_dir,
229 run_on=run_on)
231 # Update latest scenario
232 gv.latest_scenario().set_config_solver(solver)
233 gv.latest_scenario().set_config_instance_set_train(instance_set_train.directory)
234 gv.latest_scenario().set_latest_scenario(Scenario.CONFIGURATION)
236 if instance_set_test is not None:
237 gv.latest_scenario().set_config_instance_set_test(instance_set_test.directory)
238 else:
239 # Set to default to overwrite possible old path
240 gv.latest_scenario().set_config_instance_set_test()
242 # Set validation to wait until configuration is done
243 if validate:
244 validate_jobid = run_after(
245 solver, instance_set_train, instance_set_test, dependency_job_list,
246 command=CommandName.VALIDATE_CONFIGURED_VS_DEFAULT, run_on=run_on
247 )
248 dependency_job_list.append(validate_jobid)
250 if ablation:
251 ablation_jobid = run_after(
252 solver, instance_set_train, instance_set_test, dependency_job_list,
253 command=CommandName.RUN_ABLATION, run_on=run_on
254 )
255 dependency_job_list.append(ablation_jobid)
257 if run_on == Runner.SLURM:
258 job_id_str = ",".join([run.run_id for run in dependency_job_list])
259 print(f"Running configuration. Waiting for Slurm job(s) with id(s): "
260 f"{job_id_str}")
261 else:
262 print("Running configuration finished!")
264 # Write used settings to file
265 gv.settings().write_used_settings()
266 # Write used scenario to file
267 gv.latest_scenario().write_scenario_ini()