Coverage for sparkle/CLI/validate_configured_vs_default.py: 0%

1#!/usr/bin/env python3 

2"""Sparkle command to validate a configured solver against its default configuration.""" 

3 

4import sys 

5import argparse 

6from pathlib import PurePath 

7 

8from runrunner.base import Runner 

9 

10from sparkle.CLI.help import global_variables as gv 

11from sparkle.CLI.help import logging as sl 

12from sparkle.platform.settings_objects import Settings, SettingState 

13from sparkle.CLI.help import argparse_custom as ac 

14from sparkle.CLI.help.reporting_scenario import Scenario 

15from sparkle.configurator.configurator import Configurator 

16from sparkle.solver.validator import Validator 

17from sparkle.solver import Solver 

18from sparkle.instance import instance_set 

19from sparkle.platform import CommandName, COMMAND_DEPENDENCIES 

20from sparkle.CLI.initialise import check_for_initialise 

21from sparkle.CLI.help.nicknames import resolve_object_name 

22 

23 

24def parser_function() -> argparse.ArgumentParser: 

25 """Define the command line arguments.""" 

26 parser = argparse.ArgumentParser( 

27 description=("Test the performance of the configured solver and the default " 

28 "solver by doing validation experiments on the training and test " 

29 "sets.")) 

30 parser.add_argument(*ac.SolverArgument.names, 

31 **ac.SolverArgument.kwargs) 

32 parser.add_argument(*ac.InstanceSetTrainArgument.names, 

33 **ac.InstanceSetTrainArgument.kwargs) 

34 parser.add_argument(*ac.InstanceSetTestArgument.names, 

35 **ac.InstanceSetTestArgument.kwargs) 

36 parser.add_argument(*ac.ConfiguratorArgument.names, 

37 **ac.ConfiguratorArgument.kwargs) 

38 parser.add_argument(*ac.SparkleObjectiveArgument.names, 

39 **ac.SparkleObjectiveArgument.kwargs) 

40 parser.add_argument(*ac.TargetCutOffTimeValidationArgument.names, 

41 **ac.TargetCutOffTimeValidationArgument.kwargs) 

42 parser.add_argument(*ac.SettingsFileArgument.names, 

43 **ac.SettingsFileArgument.kwargs) 

44 parser.add_argument(*ac.RunOnArgument.names, 

45 **ac.RunOnArgument.kwargs) 

46 return parser 

47 

48 

49if __name__ == "__main__": 

50 # Log command call 

51 sl.log_command(sys.argv) 

52 

53 parser = parser_function() 

54 

55 # Process command line arguments 

56 args = parser.parse_args() 

57 solver = resolve_object_name(args.solver, 

58 gv.solver_nickname_mapping, 

59 gv.settings().DEFAULT_solver_dir, 

60 Solver) 

61 instance_set_train = resolve_object_name( 

62 args.instance_set_train, 

63 gv.file_storage_data_mapping[gv.instances_nickname_path], 

64 gv.settings().DEFAULT_instance_dir, instance_set) 

65 instance_set_test = resolve_object_name( 

66 args.instance_set_test, 

67 gv.file_storage_data_mapping[gv.instances_nickname_path], 

68 gv.settings().DEFAULT_instance_dir, instance_set) 

69 

70 if args.run_on is not None: 

71 gv.settings().set_run_on( 

72 args.run_on.value, SettingState.CMD_LINE) 

73 run_on = gv.settings().get_run_on() 

74 

75 check_for_initialise( 

76 COMMAND_DEPENDENCIES[CommandName.VALIDATE_CONFIGURED_VS_DEFAULT] 

77 ) 

78 if args.configurator is not None: 

79 gv.settings().set_general_sparkle_configurator( 

80 value=getattr(Configurator, args.configurator), 

81 origin=SettingState.CMD_LINE) 

82 if ac.set_by_user(args, "settings_file"): 

83 gv.settings().read_settings_ini( 

84 args.settings_file, SettingState.CMD_LINE 

85 ) # Do first, so other command line options can override settings from the file 

86 

87 if args.objectives is not None: 

88 gv.settings().set_general_sparkle_objectives( 

89 args.objectives, SettingState.CMD_LINE 

90 ) 

91 if ac.set_by_user(args, "target_cutoff_time"): 

92 gv.settings().set_general_target_cutoff_time( 

93 args.target_cutoff_time, SettingState.CMD_LINE 

94 ) 

95 # Compare current settings to latest.ini 

96 prev_settings = Settings(PurePath("Settings/latest.ini")) 

97 Settings.check_settings_changes(gv.settings(), prev_settings) 

98 

99 # Make sure configuration results exist before trying to work with them 

100 configurator = gv.settings().get_general_sparkle_configurator() 

101 configurator.set_scenario_dirs(solver, instance_set_train) 

102 objectives = gv.settings().get_general_sparkle_objectives() 

103 # Record optimised configuration 

104 _, opt_config_str = configurator.get_optimal_configuration( 

105 solver, instance_set_train, objectives[0]) 

106 opt_config = Solver.config_str_to_dict(opt_config_str) 

107 

108 validator = Validator(gv.settings().DEFAULT_validation_output, sl.caller_log_dir) 

109 all_validation_instances = [instance_set_train] 

110 if instance_set_test is not None: 

111 all_validation_instances.append(instance_set_test) 

112 validation = validator.validate( 

113 solvers=[solver] * 2, 

114 configurations=[None, opt_config], 

115 instance_sets=all_validation_instances, 

116 objectives=objectives, 

117 cut_off=gv.settings().get_general_target_cutoff_time(), 

118 sbatch_options=gv.settings().get_slurm_extra_options(as_args=True), 

119 run_on=run_on) 

120 

121 if run_on == Runner.LOCAL: 

122 validation.wait() 

123 print("Running validation done!") 

124 else: 

125 print(f"Running validation through Slurm with job ID: {validation.run_id}") 

126 

127 # Update latest scenario 

128 gv.latest_scenario().set_config_solver(solver) 

129 gv.latest_scenario().set_config_instance_set_train(instance_set_train.directory) 

130 gv.latest_scenario().set_latest_scenario(Scenario.CONFIGURATION) 

131 

132 if instance_set_test is not None: 

133 gv.latest_scenario().set_config_instance_set_test(instance_set_test.directory) 

134 else: 

135 # Set to default to overwrite possible old path 

136 gv.latest_scenario().set_config_instance_set_test() 

137 

138 # Write used settings to file 

139 gv.settings().write_used_settings() 

140 # Write used scenario to file 

141 gv.latest_scenario().write_scenario_ini()