Coverage for sparkle/solver/solver_cli.py: 0%

1#!/usr/bin/env python3 

2# -*- coding: UTF-8 -*- 

3"""Run a solver, read/write to performance dataframe.""" 

4from filelock import FileLock 

5import argparse 

6from pathlib import Path 

7import random 

8import ast 

9import time 

10 

11from runrunner import Runner 

12 

13from sparkle.solver import Solver 

14from sparkle.instance import Instance_Set 

15from sparkle.types import resolve_objective 

16from sparkle.structures import PerformanceDataFrame 

17 

18 

19if __name__ == "__main__": 

20 # Define command line arguments 

21 parser = argparse.ArgumentParser() 

22 parser.add_argument("--performance-dataframe", required=True, type=Path, 

23 help="path to the performance dataframe") 

24 parser.add_argument("--solver", required=True, type=Path, help="path to solver") 

25 parser.add_argument("--instance", required=True, type=str, 

26 help="path to instance to run on") 

27 parser.add_argument("--run-index", required=True, type=int, 

28 help="run index in the dataframe.") 

29 parser.add_argument("--log-dir", type=Path, required=True, 

30 help="path to the log directory") 

31 parser.add_argument("--configuration", type=dict, required=False, 

32 help="configuration for the solver. If not provided, read from " 

33 "the PerformanceDataFrame.") 

34 parser.add_argument("--seed", type=str, required=False, 

35 help="seed to use for the solver. If not provided, read from " 

36 "the PerformanceDataFrame or generate one.") 

37 parser.add_argument("--cutoff-time", type=int, required=False, 

38 help="the cutoff time for the solver.") 

39 parser.add_argument("--target-objective", required=False, type=str, 

40 help="The objective to use to determine the best configuration.") 

41 parser.add_argument("--best-configuration-instances", required=False, type=str, 

42 help="If given, will ignore any given configurations, and try to" 

43 " determine the best found configurations over the given " 

44 "instances. Defaults to the first objective given in the " 

45 " objectives argument or the one given by the dataframe " 

46 " to use to determine the best configuration.") 

47 args = parser.parse_args() 

48 # Process command line arguments 

49 log_dir = args.log_dir 

50 print(f"Running Solver and read/writing results with {args.performance_dataframe}") 

51 # Resolve possible multi-file instance 

52 instance_path = Path(args.instance) 

53 instance_name = instance_path.name 

54 instance_key = instance_path 

55 run_index = args.run_index 

56 if not instance_path.exists(): 

57 # If its an instance name (Multi-file instance), retrieve path list 

58 data_set = Instance_Set(instance_path.parent) 

59 instance_path = data_set.get_path_by_name(instance_name) 

60 instance_key = instance_name 

61 

62 solver = Solver(args.solver) 

63 

64 if not args.configuration or not args.seed: # Read 

65 # Desyncronize from other possible jobs writing to the same file 

66 time.sleep(random.random() * 10) 

67 lock = FileLock(f"{args.performance_dataframe}.lock") # Lock the file 

68 with lock.acquire(timeout=600): 

69 performance_dataframe = PerformanceDataFrame(args.performance_dataframe) 

70 

71 objectives = performance_dataframe.objectives 

72 # Filter out possible errors, shouldn't occur 

73 objectives = [o for o in objectives if o is not None] 

74 if args.best_configuration_instances: # Determine best configuration 

75 best_configuration_instances = args.best_configuration_instances.split(",") 

76 target_objective = resolve_objective(args.target_objective) 

77 configuration, value = performance_dataframe.best_configuration( 

78 solver=str(args.solver), 

79 objective=target_objective, 

80 instances=best_configuration_instances, 

81 ) 

82 # Read the seed from the dataframe 

83 seed = performance_dataframe.get_value( 

84 str(args.solver), 

85 str(args.instance), 

86 objective=target_objective.name, 

87 run=run_index, 

88 solver_fields=[PerformanceDataFrame.column_seed]) 

89 else: 

90 target = performance_dataframe.get_value( 

91 str(args.solver), 

92 str(args.instance), 

93 objective=None, 

94 run=run_index, 

95 solver_fields=[PerformanceDataFrame.column_seed, 

96 PerformanceDataFrame.column_configuration]) 

97 if isinstance(target[0], list): # We take the first value we find 

98 target = target[0] 

99 seed, df_configuration = target 

100 configuration = args.configuration 

101 if configuration is None: # Try to read from the dataframe 

102 if not isinstance(df_configuration, dict): 

103 try: 

104 configuration = ast.literal_eval(df_configuration) 

105 except Exception: 

106 print("Failed to read configuration from dataframe: " 

107 f"{df_configuration}") 

108 else: 

109 configuration = df_configuration 

110 seed = args.seed or seed 

111 # If no seed is provided and no seed can be read, generate one 

112 if not isinstance(seed, int): 

113 seed = random.randint(0, 2**32 - 1) 

114 

115 print(f"Running Solver {solver} on instance {instance_path.name} with seed {seed}..") 

116 solver_output = solver.run( 

117 instance_path.absolute(), 

118 objectives=objectives, 

119 seed=seed, 

120 configuration=configuration.copy() if configuration else None, 

121 cutoff_time=args.cutoff_time, 

122 log_dir=log_dir, 

123 run_on=Runner.LOCAL) 

124 

125 # Prepare the results for the DataFrame for each objective 

126 result = [[solver_output[objective.name] for objective in objectives], 

127 [seed] * len(objectives)] 

128 solver_fields = [PerformanceDataFrame.column_value, PerformanceDataFrame.column_seed] 

129 if args.best_configuration_instances: # Need to specify the configuration 

130 result.append([configuration] * len(objectives)) 

131 solver_fields.append(PerformanceDataFrame.column_configuration) 

132 objective_values = [f"{objective.name}: {solver_output[objective.name]}" 

133 for objective in objectives] 

134 print(f"Appending value objective values: {', '.join(objective_values)}") 

135 print(f"For index: Instance {args.instance}, Run {args.run_index}") 

136 

137 # Desyncronize from other possible jobs writing to the same file 

138 time.sleep(random.random() * 10) 

139 

140 # Now that we have all the results, we can add them to the performance dataframe 

141 lock = FileLock(f"{args.performance_dataframe}.lock") # Lock the file 

142 with lock.acquire(timeout=600): 

143 performance_dataframe = PerformanceDataFrame(args.performance_dataframe) 

144 performance_dataframe.set_value( 

145 result, 

146 solver=str(args.solver), 

147 instance=str(args.instance), 

148 objective=[o.name for o in objectives], 

149 run=run_index, 

150 solver_fields=solver_fields, 

151 append_write_csv=True)