Coverage for src/sparkle/solver/solver_cli.py: 73%
81 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-08 12:00 +0000
1#!/usr/bin/env python3
2# -*- coding: UTF-8 -*-
3"""Run a solver, read/write to performance dataframe."""
5import sys
6import ast
7from filelock import FileLock
8import argparse
9from pathlib import Path
10import random
11import time
13from runrunner import Runner
15from sparkle.solver import Solver
16from sparkle.instance import resolve_instance_pair
17from sparkle.types import resolve_objective
18from sparkle.structures import PerformanceDataFrame
19from sparkle.tools.solver_wrapper_parsing import parse_commandline_dict
22def main(argv: list[str]) -> None:
23 """Main function of the command."""
24 # Define command line arguments
25 parser = argparse.ArgumentParser()
26 parser.add_argument(
27 "--performance-dataframe",
28 required=True,
29 type=Path,
30 help="path to the performance dataframe",
31 )
32 parser.add_argument("--solver", required=True, type=Path, help="path to solver")
33 parser.add_argument(
34 "--instance",
35 required=True,
36 type=Path,
37 nargs="+",
38 help="path to instance to run on",
39 )
40 parser.add_argument(
41 "--run-index",
42 required=True,
43 type=int,
44 help="run index in the dataframe to set.",
45 )
46 parser.add_argument(
47 "--log-dir", type=Path, required=True, help="path to the log directory"
48 )
50 # These two arguments should be mutually exclusive
51 parser.add_argument(
52 "--configuration-id",
53 type=str,
54 required=False,
55 help="configuration id to read from the PerformanceDataFrame.",
56 )
57 parser.add_argument(
58 "--configuration",
59 type=str,
60 nargs="+",
61 required=False,
62 help="configuration for the solver",
63 )
65 parser.add_argument(
66 "--seed",
67 type=int,
68 required=False,
69 help="seed to use for the solver. If not provided, generates one.",
70 )
71 parser.add_argument(
72 "--cutoff-time",
73 type=int,
74 required=False,
75 help="the cutoff time for the solver.",
76 )
77 parser.add_argument(
78 "--objectives",
79 type=str,
80 required=False,
81 nargs="+",
82 help="The objectives to evaluate to Solver on. If not provided, read from the PerformanceDataFrame.",
83 )
84 parser.add_argument(
85 "--target-objective",
86 required=False,
87 type=str,
88 help="The objective to use to determine the best configuration.",
89 )
90 parser.add_argument(
91 "--best-configuration-instances",
92 required=False,
93 type=str,
94 nargs="+",
95 metavar="SET_NAME,INSTANCE_NAME",
96 help="If given, will ignore any given configurations, and try to"
97 " determine the best found configurations over the given "
98 "instances, each passed as a 'set_name,instance_name' pair. Uses the"
99 " 'target-objective' given in the arguments or the first one given by"
100 " the dataframe to determine the best configuration.",
101 )
102 args = parser.parse_args(argv)
103 # Process command line arguments
104 log_dir = args.log_dir
105 print(f"Running Solver and read/writing results with {args.performance_dataframe}")
106 # Resolve possible multi-file instance
107 instance_path: list[Path] = args.instance
108 # The PerformanceDataFrame is keyed by the canonical (set_name, instance_name) pair.
109 # Deriving the name from the path with .stem is only correct for FileInstanceSet, so
110 # resolve it from the owning set instead and let the subclass supply its convention.
111 # All files of a multi-file instance share the same pair, so the first file resolves it.
112 instance_pair = resolve_instance_pair(instance_path[0])
113 instance_set_name, instance_name = instance_pair
114 # If instance is only one file then we don't need a list
115 instance_path = instance_path[0] if len(instance_path) == 1 else instance_path
116 run_index = args.run_index
117 # Ensure stringifcation of path objects
118 if isinstance(instance_path, list):
119 # Double list because of solver.run
120 run_instances = [[str(filepath) for filepath in instance_path]]
121 else:
122 run_instances = str(instance_path)
124 solver = Solver(args.solver)
125 # By default, run the default configuration
126 config_id = PerformanceDataFrame.default_configuration
127 configuration = None
128 # If no seed is provided by CLI, generate one
129 seed = args.seed if args.seed else random.randint(0, 2**32 - 1)
130 # Parse the provided objectives if present
131 objectives = (
132 [resolve_objective(objective) for objective in args.objectives]
133 if args.objectives
134 else None
135 )
137 if args.configuration: # Configuration provided, override
138 if isinstance(args.configuration, list):
139 configuration = parse_commandline_dict(args.configuration)
140 else:
141 configuration = ast.literal_eval(args.configuration)
142 print(configuration)
143 config_id = configuration["configuration_id"]
144 elif (
145 (
146 args.configuration_id
147 and args.configuration_id != PerformanceDataFrame.default_configuration
148 )
149 or args.best_configuration_instances
150 or not objectives
151 ): # Read from PerformanceDataFrame, can be slow
152 # Desyncronize from other possible jobs writing to the same file
153 print(
154 "Reading from Performance DataFrame.. "
155 f"[{'configuration' if (args.configuration_id or args.best_configuration_instances) else ''} "
156 f"{'objectives' if not objectives else ''}]"
157 )
158 time.sleep(random.random() * 10)
159 lock = FileLock(f"{args.performance_dataframe}.lock") # Lock the file
160 with lock.acquire(timeout=600):
161 performance_dataframe = PerformanceDataFrame(args.performance_dataframe)
163 if not objectives:
164 objectives = performance_dataframe.objectives
166 if args.best_configuration_instances: # Determine best configuration
167 # Each token is a 'set_name,instance_name' pair, split into a tuple on the comma
168 best_configuration_instances: list[tuple[str, str]] = list(
169 {tuple(pair.split(",")) for pair in args.best_configuration_instances}
170 )
171 target_objective = (
172 resolve_objective(args.target_objective)
173 if args.target_objective
174 else objectives[0]
175 )
176 config_id, _ = performance_dataframe.best_configuration(
177 solver=str(args.solver),
178 objective=target_objective,
179 instance_pairs=best_configuration_instances,
180 )
181 configuration = performance_dataframe.get_full_configuration(
182 str(args.solver), config_id
183 )
185 elif (
186 args.configuration_id
187 ): # Read from PerformanceDataFrame the configuration using the ID
188 config_id = args.configuration_id
189 configuration = performance_dataframe.get_full_configuration(
190 str(args.solver), config_id
191 )
193 print(f"Running Solver {solver} on instance {instance_name} with seed {seed}..")
194 solver_output = solver.run(
195 run_instances,
196 objectives=objectives,
197 seed=seed,
198 configuration=configuration.copy() if configuration else None,
199 cutoff_time=args.cutoff_time,
200 log_dir=log_dir,
201 run_on=Runner.LOCAL,
202 )
204 # Prepare the results for the DataFrame for each objective
205 result = [
206 [solver_output[objective.name] for objective in objectives],
207 [seed] * len(objectives),
208 ]
209 solver_fields = [
210 PerformanceDataFrame.column_value,
211 PerformanceDataFrame.column_seed,
212 ]
214 print(f"For Solver/config: {solver}/{config_id}")
215 print(f"For index: Instance {instance_name}, Run {args.run_index}, Seed {seed}")
216 print("Appending the following objective values:") # {', '.join(objective_values)}")
217 for objective in objectives:
218 print(
219 f"{objective.name}, {instance_set_name}, {instance_name}, {args.run_index} | {args.solver}, {config_id}: {solver_output[objective.name]}"
220 )
222 # Desyncronize from other possible jobs writing to the same file
223 time.sleep(random.random() * 100)
225 # Now that we have all the results, we can add them to the performance dataframe
226 lock = FileLock(f"{args.performance_dataframe}.lock") # Lock the file
227 with lock.acquire(timeout=600):
228 performance_dataframe = PerformanceDataFrame(args.performance_dataframe)
229 performance_dataframe.set_value(
230 result,
231 solver=str(args.solver),
232 instance_pair=instance_pair,
233 configuration=config_id,
234 objective=[objective.name for objective in objectives],
235 run=run_index,
236 solver_fields=solver_fields,
237 append_write_csv=True, # We do not have to save the PDF here, thanks to this argument
238 )
241if __name__ == "__main__":
242 main(sys.argv[1:])