Coverage for sparkle/CLI/run_ablation.py: 0%

1#!/usr/bin/env python3 

2"""Sparkle command to execute ablation analysis.""" 

3 

4import argparse 

5import sys 

6from pathlib import PurePath 

7 

8from runrunner.base import Runner 

9 

10from sparkle.solver.ablation import AblationScenario 

11from sparkle.CLI.help import global_variables as gv 

12from sparkle.CLI.help import logging as sl 

13from sparkle.platform.settings_objects import Settings, SettingState 

14from sparkle.solver import Solver 

15from sparkle.instance import instance_set 

16from sparkle.CLI.help import argparse_custom as ac 

17from sparkle.platform import CommandName, COMMAND_DEPENDENCIES 

18from sparkle.CLI.initialise import check_for_initialise 

19from sparkle.CLI.help.nicknames import resolve_object_name 

20 

21 

22def parser_function() -> argparse.ArgumentParser: 

23 """Define the command line arguments.""" 

24 parser = argparse.ArgumentParser( 

25 description=("Runs parameter importance between the default and configured " 

26 "parameters with ablation. This command requires a finished " 

27 "configuration for the solver instance pair."), 

28 epilog=("Note that if no test instance set is given, the validation is performed" 

29 " on the training set.")) 

30 parser.add_argument("--solver", required=False, type=str, help="path to solver") 

31 parser.add_argument(*ac.InstanceSetTrainAblationArgument.names, 

32 **ac.InstanceSetTrainAblationArgument.kwargs) 

33 parser.add_argument(*ac.InstanceSetTestAblationArgument.names, 

34 **ac.InstanceSetTestAblationArgument.kwargs) 

35 parser.add_argument(*ac.SparkleObjectiveArgument.names, 

36 **ac.SparkleObjectiveArgument.kwargs) 

37 parser.add_argument(*ac.TargetCutOffTimeAblationArgument.names, 

38 **ac.TargetCutOffTimeAblationArgument.kwargs) 

39 parser.add_argument(*ac.WallClockTimeArgument.names, 

40 **ac.WallClockTimeArgument.kwargs) 

41 parser.add_argument(*ac.NumberOfRunsAblationArgument.names, 

42 **ac.NumberOfRunsAblationArgument.kwargs) 

43 parser.add_argument(*ac.RacingArgument.names, 

44 **ac.RacingArgument.kwargs) 

45 parser.add_argument(*ac.SettingsFileArgument.names, 

46 **ac.SettingsFileArgument.kwargs) 

47 parser.add_argument(*ac.RunOnArgument.names, 

48 **ac.RunOnArgument.kwargs) 

49 parser.set_defaults(ablation_settings_help=False) 

50 return parser 

51 

52 

53if __name__ == "__main__": 

54 sl.log_command(sys.argv) 

55 

56 # Define command line arguments 

57 parser = parser_function() 

58 

59 # Process command line arguments 

60 args = parser.parse_args() 

61 

62 solver_path = resolve_object_name(args.solver, 

63 gv.solver_nickname_mapping, 

64 gv.settings().DEFAULT_solver_dir) 

65 solver = Solver(solver_path) 

66 instance_set_train = resolve_object_name( 

67 args.instance_set_train, 

68 gv.file_storage_data_mapping[gv.instances_nickname_path], 

69 gv.settings().DEFAULT_instance_dir, instance_set) 

70 instance_set_test = resolve_object_name( 

71 args.instance_set_test, 

72 gv.file_storage_data_mapping[gv.instances_nickname_path], 

73 gv.settings().DEFAULT_instance_dir, instance_set) 

74 

75 if args.run_on is not None: 

76 gv.settings().set_run_on( 

77 args.run_on.value, SettingState.CMD_LINE) 

78 run_on = gv.settings().get_run_on() 

79 

80 check_for_initialise(COMMAND_DEPENDENCIES[CommandName.RUN_ABLATION]) 

81 

82 if ac.set_by_user(args, "settings_file"): 

83 # Do first, so other command line options can override settings from the file 

84 gv.settings().read_settings_ini( 

85 args.settings_file, SettingState.CMD_LINE 

86 ) 

87 if ac.set_by_user(args, "objectives"): 

88 gv.settings().set_general_sparkle_objectives( 

89 args.objectives, SettingState.CMD_LINE 

90 ) 

91 if ac.set_by_user(args, "target_cutoff_time"): 

92 gv.settings().set_general_target_cutoff_time( 

93 args.target_cutoff_time, SettingState.CMD_LINE 

94 ) 

95 if ac.set_by_user(args, "wallclock_time"): 

96 gv.settings().set_config_wallclock_time( 

97 args.wallclock_time, SettingState.CMD_LINE 

98 ) 

99 if ac.set_by_user(args, "number_of_runs"): 

100 gv.settings().set_config_number_of_runs( 

101 args.number_of_runs, SettingState.CMD_LINE 

102 ) 

103 if ac.set_by_user(args, "racing"): 

104 gv.settings().set_ablation_racing_flag( 

105 args.number_of_runs, SettingState.CMD_LINE 

106 ) 

107 

108 # Compare current settings to latest.ini 

109 prev_settings = Settings(PurePath("Settings/latest.ini")) 

110 Settings.check_settings_changes(gv.settings(), prev_settings) 

111 

112 instance_set_train_name = instance_set_train.name 

113 configurator = gv.settings().get_general_sparkle_configurator() 

114 configurator.set_scenario_dirs(solver, instance_set_train) 

115 if instance_set_test is not None: 

116 instance_set_test_name = instance_set_test.name 

117 else: 

118 instance_set_test = instance_set_train 

119 instance_set_test_name = instance_set_train.name 

120 

121 if not configurator.scenario.result_directory.is_dir(): 

122 print("Error: No configuration results found for the given solver and training" 

123 " instance set. Ablation needs to have a target configuration.") 

124 print("Please run configuration first") 

125 sys.exit(-1) 

126 else: 

127 print("Configuration exists!") 

128 

129 ablation_scenario = AblationScenario( 

130 solver, instance_set_train, instance_set_test, 

131 gv.settings().DEFAULT_ablation_output, 

132 gv.settings().DEFAULT_ablation_exec, 

133 gv.settings().DEFAULT_ablation_validation_exec, override_dirs=True) 

134 

135 # Instances 

136 ablation_scenario.create_instance_file() 

137 ablation_scenario.create_instance_file(test=True) 

138 

139 # Configurations 

140 ablation_scenario.create_configuration_file() 

141 print("Submiting ablation run...") 

142 runs = ablation_scenario.submit_ablation(run_on=run_on) 

143 

144 if run_on == Runner.LOCAL: 

145 print("Ablation analysis finished!") 

146 else: 

147 job_id_str = ",".join([run.run_id for run in runs]) 

148 print(f"Ablation analysis running. Waiting for Slurm job(s) with id(s): " 

149 f"{job_id_str}")