Coverage for sparkle/CLI/validate_configured_vs_default.py: 89%
75 statements
« prev ^ index » next coverage.py v7.6.4, created at 2024-11-05 14:48 +0000
1#!/usr/bin/env python3
2"""Sparkle command to validate a configured solver against its default configuration."""
4import sys
5import argparse
6from pathlib import PurePath
8from runrunner.base import Runner
10from sparkle.CLI.help import global_variables as gv
11from sparkle.CLI.help import logging as sl
12from sparkle.platform.settings_objects import Settings, SettingState
13from sparkle.CLI.help import argparse_custom as ac
14from sparkle.CLI.help.reporting_scenario import Scenario
15from sparkle.configurator.configurator import Configurator
16from sparkle.solver.validator import Validator
17from sparkle.solver import Solver
18from sparkle.instance import Instance_Set
19from sparkle.platform import CommandName, COMMAND_DEPENDENCIES
20from sparkle.CLI.initialise import check_for_initialise
21from sparkle.CLI.help.nicknames import resolve_object_name
24def parser_function() -> argparse.ArgumentParser:
25 """Define the command line arguments."""
26 parser = argparse.ArgumentParser(
27 description="Test the performance of the configured solver and the default "
28 "solver by doing validation experiments on the training and test "
29 "sets.")
30 parser.add_argument(*ac.SolverArgument.names,
31 **ac.SolverArgument.kwargs)
32 parser.add_argument(*ac.InstanceSetTrainArgument.names,
33 **ac.InstanceSetTrainArgument.kwargs)
34 parser.add_argument(*ac.InstanceSetTestArgument.names,
35 **ac.InstanceSetTestArgument.kwargs)
36 parser.add_argument(*ac.ConfiguratorArgument.names,
37 **ac.ConfiguratorArgument.kwargs)
38 parser.add_argument(*ac.SparkleObjectiveArgument.names,
39 **ac.SparkleObjectiveArgument.kwargs)
40 parser.add_argument(*ac.TargetCutOffTimeValidationArgument.names,
41 **ac.TargetCutOffTimeValidationArgument.kwargs)
42 parser.add_argument(*ac.SettingsFileArgument.names,
43 **ac.SettingsFileArgument.kwargs)
44 parser.add_argument(*ac.RunOnArgument.names,
45 **ac.RunOnArgument.kwargs)
46 return parser
49def main(argv: list[str]) -> None:
50 """Run the validate configured vs default command."""
51 # Log command call
52 sl.log_command(sys.argv)
54 parser = parser_function()
56 # Process command line arguments
57 args = parser.parse_args(argv)
59 check_for_initialise(
60 COMMAND_DEPENDENCIES[CommandName.VALIDATE_CONFIGURED_VS_DEFAULT]
61 )
63 if ac.set_by_user(args, "settings_file"):
64 gv.settings().read_settings_ini(
65 args.settings_file, SettingState.CMD_LINE
66 ) # Do first, so other command line options can override settings from the file
67 if args.configurator is not None:
68 gv.settings().set_general_sparkle_configurator(
69 value=getattr(Configurator, args.configurator),
70 origin=SettingState.CMD_LINE)
71 if args.objectives is not None:
72 gv.settings().set_general_sparkle_objectives(
73 args.objectives, SettingState.CMD_LINE
74 )
75 if ac.set_by_user(args, "target_cutoff_time"):
76 gv.settings().set_general_target_cutoff_time(
77 args.target_cutoff_time, SettingState.CMD_LINE
78 )
79 if args.run_on is not None:
80 gv.settings().set_run_on(
81 args.run_on.value, SettingState.CMD_LINE)
83 # Compare current settings to latest.ini
84 prev_settings = Settings(PurePath("Settings/latest.ini"))
85 Settings.check_settings_changes(gv.settings(), prev_settings)
87 run_on = gv.settings().get_run_on()
89 solver = resolve_object_name(args.solver,
90 gv.solver_nickname_mapping,
91 gv.settings().DEFAULT_solver_dir,
92 Solver)
93 instance_set_train = resolve_object_name(
94 args.instance_set_train,
95 gv.file_storage_data_mapping[gv.instances_nickname_path],
96 gv.settings().DEFAULT_instance_dir, Instance_Set)
97 instance_set_test = resolve_object_name(
98 args.instance_set_test,
99 gv.file_storage_data_mapping[gv.instances_nickname_path],
100 gv.settings().DEFAULT_instance_dir, Instance_Set)
102 # Make sure configuration results exist before trying to work with them
103 configurator = gv.settings().get_general_sparkle_configurator()
104 config_scenario = configurator.scenario_class.find_scenario(
105 configurator.output_path, solver, instance_set_train)
107 if config_scenario is None:
108 print("No configuration scenario found for combination:\n"
109 f"{configurator.name} {solver.name} {instance_set_train.name}")
110 sys.exit(-1)
112 # Record optimised configuration
113 _, opt_config_str = configurator.get_optimal_configuration(config_scenario)
114 opt_config = Solver.config_str_to_dict(opt_config_str)
116 validator = Validator(gv.settings().DEFAULT_validation_output, sl.caller_log_dir)
117 all_validation_instances = [instance_set_train]
118 if instance_set_test is not None:
119 all_validation_instances.append(instance_set_test)
120 validation = validator.validate(
121 solvers=[solver] * 2,
122 configurations=[None, opt_config],
123 instance_sets=all_validation_instances,
124 objectives=config_scenario.sparkle_objectives,
125 cut_off=gv.settings().get_general_target_cutoff_time(),
126 sbatch_options=gv.settings().get_slurm_extra_options(as_args=True),
127 run_on=run_on)
129 if run_on == Runner.LOCAL:
130 validation.wait()
131 print("Running validation done!")
132 else:
133 print(f"Running validation through Slurm with job ID: {validation.run_id}")
135 # Update latest scenario
136 gv.latest_scenario().set_config_solver(solver)
137 gv.latest_scenario().set_config_instance_set_train(instance_set_train.directory)
138 gv.latest_scenario().set_latest_scenario(Scenario.CONFIGURATION)
140 if instance_set_test is not None:
141 gv.latest_scenario().set_config_instance_set_test(instance_set_test.directory)
142 else:
143 # Set to default to overwrite possible old path
144 gv.latest_scenario().set_config_instance_set_test()
146 # Write used settings to file
147 gv.settings().write_used_settings()
148 # Write used scenario to file
149 gv.latest_scenario().write_scenario_ini()
150 sys.exit(0)
153if __name__ == "__main__":
154 main(sys.argv[1:])