Coverage for sparkle/CLI/run_ablation.py: 74%

1#!/usr/bin/env python3 

2"""Sparkle command to execute ablation analysis.""" 

3 

4import argparse 

5import sys 

6from pathlib import PurePath 

7 

8from runrunner.base import Runner 

9 

10from sparkle.configurator import AblationScenario 

11from sparkle.CLI.help import global_variables as gv 

12from sparkle.CLI.help import logging as sl 

13from sparkle.platform.settings_objects import Settings, SettingState 

14from sparkle.solver import Solver 

15from sparkle.structures import PerformanceDataFrame 

16from sparkle.instance import Instance_Set, InstanceSet 

17from sparkle.CLI.help import argparse_custom as ac 

18from sparkle.CLI.initialise import check_for_initialise 

19from sparkle.CLI.help.nicknames import resolve_object_name 

20 

21 

22def parser_function() -> argparse.ArgumentParser: 

23 """Define the command line arguments.""" 

24 parser = argparse.ArgumentParser( 

25 description="Runs parameter importance between the default and configured " 

26 "parameters with ablation. This command requires a finished " 

27 "configuration for the solver instance pair.", 

28 epilog="Note that if no test instance set is given, the validation is performed" 

29 " on the training set.") 

30 parser.add_argument("--solver", required=False, type=str, help="path to solver") 

31 parser.add_argument(*ac.InstanceSetTrainOptionalArgument.names, 

32 **ac.InstanceSetTrainOptionalArgument.kwargs) 

33 parser.add_argument(*ac.InstanceSetTestAblationArgument.names, 

34 **ac.InstanceSetTestAblationArgument.kwargs) 

35 parser.add_argument(*ac.ObjectivesArgument.names, 

36 **ac.ObjectivesArgument.kwargs) 

37 parser.add_argument(*ac.SolverCutOffTimeArgument.names, 

38 **ac.SolverCutOffTimeArgument.kwargs) 

39 parser.add_argument(*ac.WallClockTimeArgument.names, 

40 **ac.WallClockTimeArgument.kwargs) 

41 parser.add_argument(*ac.NumberOfRunsAblationArgument.names, 

42 **ac.NumberOfRunsAblationArgument.kwargs) 

43 parser.add_argument(*ac.RacingArgument.names, 

44 **ac.RacingArgument.kwargs) 

45 parser.add_argument(*ac.SettingsFileArgument.names, 

46 **ac.SettingsFileArgument.kwargs) 

47 parser.add_argument(*ac.RunOnArgument.names, 

48 **ac.RunOnArgument.kwargs) 

49 parser.set_defaults(ablation_settings_help=False) 

50 return parser 

51 

52 

53def main(argv: list[str]) -> None: 

54 """Main function to run ablation analysis.""" 

55 sl.log_command(sys.argv) 

56 check_for_initialise() 

57 

58 if not AblationScenario.check_requirements(verbose=True): 

59 print("Ablation Analysis is not available.") 

60 if not AblationScenario.ablation_executable.exists(): 

61 print("Would you like to download it? (Y/n)") 

62 if input().lower().strip() == "y": 

63 AblationScenario.download_requirements() 

64 else: 

65 sys.exit() 

66 else: 

67 print("Check that Java is available on your system.") 

68 sys.exit(-1) 

69 

70 # Define command line arguments 

71 parser = parser_function() 

72 

73 # Process command line arguments 

74 args = parser.parse_args(argv) 

75 

76 if args.settings_file is not None: 

77 # Do first, so other command line options can override settings from the file 

78 gv.settings().read_settings_ini( 

79 args.settings_file, SettingState.CMD_LINE 

80 ) 

81 if args.objectives is not None: 

82 gv.settings().set_general_sparkle_objectives( 

83 args.objectives, SettingState.CMD_LINE 

84 ) 

85 if args.solver_cutoff_time is not None: 

86 gv.settings().set_general_solver_cutoff_time( 

87 args.solver_cutoff_time, SettingState.CMD_LINE 

88 ) 

89 if args.wallclock_time is not None: 

90 gv.settings().set_smac2_wallclock_time( 

91 args.wallclock_time, SettingState.CMD_LINE 

92 ) 

93 if args.number_of_runs is not None: 

94 gv.settings().set_configurator_number_of_runs( 

95 args.number_of_runs, SettingState.CMD_LINE 

96 ) 

97 if args.racing is not None: 

98 gv.settings().set_ablation_racing_flag( 

99 args.number_of_runs, SettingState.CMD_LINE 

100 ) 

101 if args.run_on is not None: 

102 gv.settings().set_run_on( 

103 args.run_on.value, SettingState.CMD_LINE) 

104 

105 # Compare current settings to latest.ini 

106 prev_settings = Settings(PurePath("Settings/latest.ini")) 

107 Settings.check_settings_changes(gv.settings(), prev_settings) 

108 

109 run_on = gv.settings().get_run_on() 

110 solver = resolve_object_name(args.solver, 

111 gv.solver_nickname_mapping, 

112 gv.settings().DEFAULT_solver_dir, Solver) 

113 if solver is None: 

114 print(f"Could not resolve Solver path/name {args.solver}!") 

115 print([p for p in gv.settings().DEFAULT_solver_dir.iterdir()]) 

116 sys.exit(-1) 

117 

118 instance_set_train: InstanceSet = resolve_object_name( 

119 args.instance_set_train, 

120 gv.file_storage_data_mapping[gv.instances_nickname_path], 

121 gv.settings().DEFAULT_instance_dir, Instance_Set) 

122 instance_set_test = resolve_object_name( 

123 args.instance_set_test, 

124 gv.file_storage_data_mapping[gv.instances_nickname_path], 

125 gv.settings().DEFAULT_instance_dir, Instance_Set) 

126 

127 configurator = gv.settings().get_general_sparkle_configurator() 

128 output_path = gv.settings().get_configurator_output_path(configurator) 

129 config_scenario = configurator.scenario_class().find_scenario( 

130 output_path, solver, instance_set_train) 

131 performance_data = PerformanceDataFrame( 

132 gv.settings().DEFAULT_performance_data_path) 

133 if config_scenario is None: 

134 print("No configuration scenario found for combination:\n" 

135 f"{configurator.name} {solver.name} {instance_set_train.name}") 

136 sys.exit(-1) 

137 best_configuration_key, _ = performance_data.best_configuration( 

138 str(solver.directory), 

139 config_scenario.sparkle_objective, 

140 instances=instance_set_train.instance_names) 

141 best_configuration = performance_data.get_full_configuration( 

142 str(solver.directory), 

143 best_configuration_key) 

144 if instance_set_test is None: 

145 instance_set_test = instance_set_train 

146 

147 if not config_scenario.results_directory.is_dir(): 

148 print("Error: No configuration results found for the given solver and training" 

149 " instance set. Ablation needs to have a target configuration. " 

150 "Please finish configuration first.") 

151 sys.exit(-1) 

152 else: 

153 print("Configuration exists!") 

154 

155 ablation_scenario = AblationScenario( 

156 config_scenario, 

157 instance_set_test, 

158 cutoff_length=gv.settings().get_smac2_target_cutoff_length(), # NOTE: SMAC2 

159 concurrent_clis=gv.settings().get_slurm_max_parallel_runs_per_node(), 

160 best_configuration=best_configuration, 

161 ablation_racing=gv.settings().get_ablation_racing_flag(), 

162 ) 

163 

164 # Create scenario 

165 ablation_scenario.create_scenario(override_dirs=True) 

166 

167 print("Submiting ablation run...") 

168 runs = ablation_scenario.submit_ablation( 

169 log_dir=sl.caller_log_dir, 

170 sbatch_options=gv.settings().get_slurm_extra_options(as_args=True), 

171 run_on=run_on) 

172 

173 if run_on == Runner.LOCAL: 

174 print("Ablation analysis finished!") 

175 else: 

176 job_id_str = ",".join([run.run_id for run in runs]) 

177 print(f"Ablation analysis running through Slurm with job id(s): " 

178 f"{job_id_str}") 

179 sys.exit(0) 

180 

181 

182if __name__ == "__main__": 

183 main(sys.argv[1:])