Coverage for src/sparkle/CLI/construct_portfolio_selector.py: 13%
135 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-08 12:00 +0000
1#!/usr/bin/env python3
2"""Sparkle command to construct a portfolio selector."""
4import sys
5import argparse
7from runrunner.base import Runner
9from sparkle.selector import Selector, SelectionScenario
10from sparkle.instance import Instance_Set
12from sparkle.platform.settings_objects import Settings
13from sparkle.structures import PerformanceDataFrame, FeatureDataFrame
14from sparkle.types import resolve_objective
15from sparkle.CLI.help import global_variables as gv
16from sparkle.CLI.help import logging as sl
17from sparkle.CLI.help import argparse_custom as ac
18from sparkle.CLI.help.nicknames import resolve_object_name
19from sparkle.CLI.initialise import check_for_initialise
22def parser_function() -> argparse.ArgumentParser:
23 """Define the command line arguments."""
24 parser = argparse.ArgumentParser(
25 description="Command to construct a portfolio selector over all known features "
26 "solver performances."
27 )
28 parser.add_argument(*ac.SolversArgument.names, **ac.SolversArgument.kwargs)
29 parser.add_argument(
30 *ac.RecomputePortfolioSelectorArgument.names,
31 **ac.RecomputePortfolioSelectorArgument.kwargs,
32 )
33 parser.add_argument(*ac.ObjectiveArgument.names, **ac.ObjectiveArgument.kwargs)
34 parser.add_argument(
35 *ac.SelectorAblationArgument.names, **ac.SelectorAblationArgument.kwargs
36 )
37 parser.add_argument(
38 *ac.InstanceSetTrainOptionalArgument.names,
39 **ac.InstanceSetTrainOptionalArgument.kwargs,
40 )
41 # Solver Configurations arguments
42 configuration_group = parser.add_mutually_exclusive_group(required=False)
43 configuration_group.add_argument(
44 *ac.AllSolverConfigurationArgument.names,
45 **ac.AllSolverConfigurationArgument.kwargs,
46 )
47 configuration_group.add_argument(
48 *ac.BestSolverConfigurationArgument.names,
49 **ac.BestSolverConfigurationArgument.kwargs,
50 )
51 configuration_group.add_argument(
52 *ac.DefaultSolverConfigurationArgument.names,
53 **ac.DefaultSolverConfigurationArgument.kwargs,
54 )
55 # TODO: Allow user to specify configuration ids to use
56 # Settings arguments
57 parser.add_argument(*ac.SettingsFileArgument.names, **ac.SettingsFileArgument.kwargs)
58 parser.add_argument(
59 *Settings.OPTION_minimum_marginal_contribution.args,
60 **Settings.OPTION_minimum_marginal_contribution.kwargs,
61 )
62 parser.add_argument(*Settings.OPTION_run_on.args, **Settings.OPTION_run_on.kwargs)
63 return parser
66def judge_exist_remaining_jobs(
67 feature_data: FeatureDataFrame, performance_data: PerformanceDataFrame
68) -> bool:
69 """Return whether there are remaining feature or performance computation jobs."""
70 missing_features = feature_data.has_missing_vectors()
71 missing_performances = performance_data.has_missing_values
72 if missing_features:
73 print(
74 "There remain unperformed feature computation jobs! Please run: "
75 "'sparkle compute features'"
76 )
77 if missing_performances:
78 print(
79 "There remain unperformed performance computation jobs! Please run:\n"
80 "'sparkle cleanup --performance-data'\n"
81 "to check for missing values in the logs, otherwise run:\n"
82 "'sparkle run solvers --performance-data'\n"
83 "to compute missing values."
84 )
85 if missing_features or missing_performances:
86 print(
87 "Please first execute all unperformed jobs before constructing Sparkle "
88 "portfolio selector."
89 )
90 sys.exit(-1)
93def main(argv: list[str]) -> None:
94 """Main method of construct portfolio selector."""
95 # Define command line arguments
96 parser = parser_function()
98 # Process command line arguments
99 args = parser.parse_args(argv)
100 settings = gv.settings(args)
102 # Log command call
103 sl.log_command(sys.argv, settings.random_state)
104 check_for_initialise()
106 flag_recompute_portfolio = args.recompute_portfolio_selector
107 solver_ablation = args.solver_ablation
109 if args.objective is not None:
110 objective = resolve_objective(args.objective)
111 else:
112 objective = settings.objectives[0]
113 print(
114 "WARNING: No objective specified, defaulting to first objective from "
115 f"settings ({objective})."
116 )
117 run_on = settings.run_on
119 print("Start constructing Sparkle portfolio selector ...")
120 if settings.selection_class is None or settings.selection_model is None:
121 print(
122 "Selector class or model not set! Values:\n"
123 f"\t - Class: {settings.selection_class}\n"
124 f"\t - Model: {settings.selection_model}\n"
125 "Please specify these values in the sparkle settings."
126 )
127 sys.exit(-1)
129 selector = Selector(settings.selection_class, settings.selection_model)
131 instance_set = None
132 if args.instance_set_train is not None:
133 instance_set = resolve_object_name(
134 args.instance_set_train,
135 gv.file_storage_data_mapping[gv.instances_nickname_path],
136 gv.settings().DEFAULT_instance_dir,
137 Instance_Set,
138 )
140 solver_cutoff_time = gv.settings().solver_cutoff_time
141 extractor_cutoff_time = gv.settings().extractor_cutoff_time
143 performance_data = PerformanceDataFrame(gv.settings().DEFAULT_performance_data_path)
144 feature_data = FeatureDataFrame(gv.settings().DEFAULT_feature_data_path)
146 # Check that the feature data actually contains features (extractors)
147 if feature_data.num_features == 0:
148 print(
149 "ERROR: Feature data is empty! Please add a feature extractor and run "
150 "'sparkle compute features' first."
151 )
152 sys.exit(-1)
154 # Filter objective
155 performance_data.remove_objective(
156 [obj for obj in performance_data.objective_names if obj != objective.name]
157 )
158 if instance_set is not None:
159 removable_instances = [
160 instance_pair
161 for instance_pair in performance_data.instance_pairs
162 if instance_pair not in instance_set.instance_pairs
163 ]
164 performance_data.remove_instance(removable_instances)
165 feature_data.remove_instance(removable_instances)
167 if args.solvers is not None:
168 solvers = args.solvers
169 removeable_solvers = [
170 solver for solver in performance_data.solvers if solver not in solvers
171 ]
172 performance_data.remove_solver(removeable_solvers)
173 else:
174 solvers = sorted(
175 [
176 str(solver)
177 for solver in gv.settings().DEFAULT_solver_dir.iterdir()
178 if solver.is_dir()
179 ]
180 )
182 # Check what configurations should be considered
183 if args.best_configuration:
184 configurations = {
185 solver: performance_data.best_configuration(solver, objective=objective)
186 for solver in solvers
187 }
188 elif args.default_configuration:
189 configurations = {
190 solver: PerformanceDataFrame.default_configuration for solver in solvers
191 }
192 else:
193 configurations = {
194 solver: performance_data.get_configurations(solver) for solver in solvers
195 }
196 if not args.all_configurations: # Take the only configuration
197 if any(len(configuration) > 1 for configuration in configurations.values()):
198 print("ERROR: More than one configuration for the following solvers:")
199 for solver, config in configurations.items():
200 if len(config) > 1:
201 print(f"\t{solver}: {config} configurations")
202 raise ValueError(
203 "Please set the --all-configurations flag if you wish to use more "
204 "than one configuration per solver."
205 )
206 for solver in solvers:
207 removeable_configs = [
208 configuration
209 for configuration in performance_data.get_configurations(solver)
210 if configuration not in configurations[solver]
211 ]
212 performance_data.remove_configuration(solver, removeable_configs)
214 judge_exist_remaining_jobs(feature_data, performance_data)
215 if feature_data.has_missing_value():
216 print(
217 "WARNING: Missing values in the feature data, will be imputed as the mean "
218 "value of all other non-missing values! Imputing all missing values..."
219 )
220 feature_data.impute_missing_values()
222 # Filter the scenario from Solver (Configurations) that do not meet the minimum marginal contribution on the training set
223 if gv.settings().minimum_marginal_contribution > 0.0:
224 print(
225 f"Filtering the scenario from Solver (Configurations) with contribution < {gv.settings().minimum_marginal_contribution} ..."
226 )
227 for (
228 solver,
229 config_id,
230 marginal_contribution,
231 _,
232 ) in performance_data.marginal_contribution(objective=objective):
233 if marginal_contribution < gv.settings().minimum_marginal_contribution:
234 print(f"\tRemoving {solver}, {config_id} [{marginal_contribution}]")
235 performance_data.remove_configuration(solver, config_id)
237 selection_scenario = SelectionScenario(
238 gv.settings().DEFAULT_selection_output,
239 selector,
240 objective,
241 performance_data,
242 feature_data,
243 solver_cutoff=solver_cutoff_time,
244 extractor_cutoff=extractor_cutoff_time,
245 ablate=solver_ablation,
246 )
248 if selection_scenario.selector_file_path.exists():
249 if not flag_recompute_portfolio:
250 print(
251 "Portfolio selector already exists. "
252 "Set the recompute flag to remove and reconstruct."
253 )
254 sys.exit(-1)
255 # Delete all selectors
256 selection_scenario.selector_file_path.unlink(missing_ok=True)
257 if selection_scenario.ablation_scenarios:
258 for scenario in selection_scenario.ablation_scenarios:
259 scenario.selector_file_path.unlink(missing_ok=True)
261 sbatch_options = gv.settings().sbatch_settings
262 slurm_prepend = gv.settings().slurm_job_prepend
263 selector_run = selector.construct(
264 selection_scenario,
265 run_on=run_on,
266 sbatch_options=sbatch_options,
267 slurm_prepend=slurm_prepend,
268 base_dir=sl.caller_log_dir,
269 )
270 jobs = [selector_run]
271 if run_on == Runner.LOCAL:
272 print("Sparkle portfolio selector constructed!")
273 else:
274 print("Sparkle portfolio selector constructor running...")
276 # Validate the selector to run on the given instances
277 instances = [
278 Settings.DEFAULT_instance_dir / instance_set_name / instance_name
279 for instance_set_name, instance_name in performance_data.instance_pairs
280 ]
281 selector_validation = selector.run_cli(
282 selection_scenario.scenario_file,
283 instances,
284 feature_data.csv_filepath,
285 run_on=run_on,
286 sbatch_options=sbatch_options,
287 slurm_prepend=slurm_prepend,
288 dependencies=[selector_run],
289 log_dir=sl.caller_log_dir,
290 )
291 jobs.append(selector_validation)
293 if solver_ablation:
294 for ablated_scenario in selection_scenario.ablation_scenarios:
295 # Construct the ablated selector
296 ablation_run = selector.construct(
297 ablated_scenario,
298 run_on=run_on,
299 sbatch_options=sbatch_options,
300 slurm_prepend=slurm_prepend,
301 base_dir=sl.caller_log_dir,
302 )
303 # Validate the ablated selector
304 ablation_validation = selector.run_cli(
305 ablated_scenario.scenario_file,
306 instances,
307 feature_data.csv_filepath,
308 run_on=run_on,
309 sbatch_options=sbatch_options,
310 slurm_prepend=slurm_prepend,
311 job_name=f"Selector Ablation: {ablated_scenario.directory.name} on {len(instances)} instances",
312 dependencies=[ablation_run],
313 log_dir=sl.caller_log_dir,
314 )
315 jobs.extend([ablation_run, ablation_validation])
317 if run_on == Runner.LOCAL:
318 for job in jobs:
319 job.wait()
320 selector_validation.wait()
321 print("Selector validation done!")
322 else:
323 print(
324 f"Running selector construction through Slurm with job id(s): "
325 f"{', '.join([d.run_id for d in jobs])}"
326 )
328 # Write used settings to file
329 gv.settings().write_used_settings()
330 sys.exit(0)
333if __name__ == "__main__":
334 main(sys.argv[1:])