Coverage for sparkle/solver/solver_cli.py: 0%

1#!/usr/bin/env python3 

2# -*- coding: UTF-8 -*- 

3"""Run a solver, read/write to performance dataframe.""" 

4from filelock import FileLock 

5import argparse 

6from pathlib import Path 

7import random 

8import time 

9 

10from runrunner import Runner 

11 

12from sparkle.solver import Solver 

13from sparkle.types import resolve_objective 

14from sparkle.structures import PerformanceDataFrame 

15 

16 

17if __name__ == "__main__": 

18 # Define command line arguments 

19 parser = argparse.ArgumentParser() 

20 parser.add_argument("--performance-dataframe", required=True, type=Path, 

21 help="path to the performance dataframe") 

22 parser.add_argument("--solver", required=True, type=Path, help="path to solver") 

23 parser.add_argument("--instance", required=True, type=Path, nargs="+", 

24 help="path to instance to run on") 

25 parser.add_argument("--run-index", required=True, type=int, 

26 help="run index in the dataframe to set.") 

27 parser.add_argument("--log-dir", type=Path, required=True, 

28 help="path to the log directory") 

29 

30 # These two arguments should be mutually exclusive 

31 parser.add_argument("--configuration-id", type=str, required=False, 

32 help="configuration id to read from the PerformanceDataFrame.") 

33 parser.add_argument("--configuration", type=dict, required=False, 

34 help="configuration for the solver") 

35 

36 parser.add_argument("--seed", type=str, required=False, 

37 help="seed to use for the solver. If not provided, read from " 

38 "the PerformanceDataFrame or generate one.") 

39 parser.add_argument("--cutoff-time", type=int, required=False, 

40 help="the cutoff time for the solver.") 

41 parser.add_argument("--target-objective", required=False, type=str, 

42 help="The objective to use to determine the best configuration.") 

43 parser.add_argument("--best-configuration-instances", 

44 required=False, type=str, nargs="+", 

45 help="If given, will ignore any given configurations, and try to" 

46 " determine the best found configurations over the given " 

47 "instances. Defaults to the first objective given in the " 

48 " objectives argument or the one given by the dataframe " 

49 " to use to determine the best configuration.") 

50 args = parser.parse_args() 

51 # Process command line arguments 

52 log_dir = args.log_dir 

53 print(f"Running Solver and read/writing results with {args.performance_dataframe}") 

54 # Resolve possible multi-file instance 

55 instance_path: list[Path] = args.instance 

56 # If instance is only one file then we don't need a list 

57 instance_path = instance_path[0] if len(instance_path) == 1 else instance_path 

58 instance_name = instance_path.stem if isinstance( 

59 instance_path, Path) else instance_path[0].stem 

60 run_index = args.run_index 

61 # Ensure stringifcation of path objects 

62 if isinstance(instance_path, list): 

63 # Double list because of solver.run 

64 run_instances = [[str(filepath) for filepath in instance_path]] 

65 else: 

66 run_instances = str(instance_path) 

67 

68 solver = Solver(args.solver) 

69 

70 if args.configuration_id or args.best_configuration_instances: # Read 

71 # Desyncronize from other possible jobs writing to the same file 

72 time.sleep(random.random() * 10) 

73 lock = FileLock(f"{args.performance_dataframe}.lock") # Lock the file 

74 with lock.acquire(timeout=600): 

75 performance_dataframe = PerformanceDataFrame(args.performance_dataframe) 

76 

77 objectives = performance_dataframe.objectives 

78 # Filter out possible errors, shouldn't occur 

79 objectives = [o for o in objectives if o is not None] 

80 if args.best_configuration_instances: # Determine best configuration 

81 best_configuration_instances: list[str] = args.best_configuration_instances 

82 # Get the unique instance names 

83 best_configuration_instances = list(set([ 

84 Path(instance).stem 

85 for instance in best_configuration_instances])) 

86 target_objective = resolve_objective(args.target_objective) 

87 config_id, value = performance_dataframe.best_configuration( 

88 solver=str(args.solver), 

89 objective=target_objective, 

90 instances=best_configuration_instances, 

91 ) 

92 configuration = performance_dataframe.get_full_configuration( 

93 str(args.solver), config_id) 

94 # Read the seed from the dataframe 

95 seed = performance_dataframe.get_value( 

96 str(args.solver), 

97 instance_name, 

98 objective=target_objective.name, 

99 run=run_index, 

100 solver_fields=[PerformanceDataFrame.column_seed]) 

101 elif args.configuration_id: # Read from DF the ID 

102 config_id = args.configuration_id 

103 configuration = performance_dataframe.get_full_configuration( 

104 str(args.solver), config_id) 

105 seed = performance_dataframe.get_value( 

106 str(args.solver), 

107 instance_name, 

108 objective=None, 

109 run=run_index, 

110 solver_fields=[PerformanceDataFrame.column_seed]) 

111 else: # Direct config given 

112 configuration = args.configuration 

113 config_id = configuration["config_id"] 

114 

115 seed = args.seed or seed 

116 # If no seed is provided and no seed can be read, generate one 

117 if not isinstance(seed, int): 

118 seed = random.randint(0, 2**32 - 1) 

119 

120 print(f"Running Solver {solver} on instance {instance_name} with seed {seed}..") 

121 solver_output = solver.run( 

122 run_instances, 

123 objectives=objectives, 

124 seed=seed, 

125 configuration=configuration.copy() if configuration else None, 

126 cutoff_time=args.cutoff_time, 

127 log_dir=log_dir, 

128 run_on=Runner.LOCAL) 

129 

130 # Prepare the results for the DataFrame for each objective 

131 result = [[solver_output[objective.name] for objective in objectives], 

132 [seed] * len(objectives)] 

133 solver_fields = [PerformanceDataFrame.column_value, PerformanceDataFrame.column_seed] 

134 objective_values = [f"{objective.name}: {solver_output[objective.name]}" 

135 for objective in objectives] 

136 print(f"Appending value objective values: {', '.join(objective_values)}") 

137 print(f"For Solver/config: {solver}/{config_id}") 

138 print(f"For index: Instance {instance_name}, Run {args.run_index}") 

139 

140 # Desyncronize from other possible jobs writing to the same file 

141 time.sleep(random.random() * 10) 

142 

143 # Now that we have all the results, we can add them to the performance dataframe 

144 lock = FileLock(f"{args.performance_dataframe}.lock") # Lock the file 

145 with lock.acquire(timeout=600): 

146 performance_dataframe = PerformanceDataFrame(args.performance_dataframe) 

147 performance_dataframe.set_value( 

148 result, 

149 solver=str(args.solver), 

150 instance=instance_name, 

151 configuration=config_id, 

152 objective=[o.name for o in objectives], 

153 run=run_index, 

154 solver_fields=solver_fields, 

155 append_write_csv=True)