Coverage for sparkle/CLI/help/argparse_custom.py: 84%
97 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-09-27 09:10 +0000
1"""Custom helper class and functions to process CLI arguments with argparse."""
3from __future__ import annotations
4import argparse
5import enum
6from pathlib import Path
7from typing import Any
9from runrunner.base import Runner
11from sparkle.platform.settings_objects import SettingState, Settings
14class SetByUser(argparse.Action):
15 """Possible action to execute for CLI argument."""
17 def __call__(self: SetByUser, parser: argparse.ArgumentParser,
18 namespace: argparse.Namespace, values: str, option_string: str = None)\
19 -> None:
20 """Set attributes when called."""
21 setattr(namespace, self.dest, values)
22 setattr(namespace, self.dest + "_nondefault", True)
25# Taken from https://stackoverflow.com/a/60750535
26class EnumAction(argparse.Action):
27 """Argparse action for handling Enums."""
28 def __init__(self: EnumAction, **kwargs: str) -> None:
29 """Initialise the EnumAction."""
30 # Pop off the type value
31 enum_type = kwargs.pop("type", None)
33 # Ensure an Enum subclass is provided
34 if enum_type is None:
35 raise ValueError("type must be assigned an Enum when using EnumAction")
36 if not issubclass(enum_type, enum.Enum):
37 raise TypeError("type must be an Enum when using EnumAction")
39 # Generate choices from the Enum
40 kwargs.setdefault("choices", tuple(e.value for e in enum_type))
42 super(EnumAction, self).__init__(**kwargs)
44 self._enum = enum_type
46 def __call__(self: EnumAction, parser: argparse.ArgumentParser,
47 namespace: argparse.Namespace, values: str, option_string: str = None) \
48 -> None:
49 """Converts value back to Enum."""
50 value = self._enum(values)
51 setattr(namespace, self.dest, value)
54def user_set_state(args: argparse.Namespace, arg_name: str) -> SettingState:
55 """Return the SettingState of an argument."""
56 if hasattr(args, arg_name + "_nondefault"):
57 return SettingState.CMD_LINE
58 else:
59 return SettingState.DEFAULT
62def set_by_user(args: argparse.Namespace, arg_name: str) -> bool:
63 """Return whether an argument was set through the CLI by the user or not."""
64 return hasattr(args, arg_name + "_nondefault")
67class ArgumentContainer():
68 """Helper class for more convenient argument packaging and access."""
69 def __init__(self: ArgumentContainer, names: list[str], kwargs: dict[str, Any])\
70 -> None:
71 """Create an ArgumentContainer.
73 Args:
74 names: List of names for the contained argument. For positional arguments,
75 this will contain a single string. For positional arguments this will
76 typically contain two, the first one starting with '-' and the second one
77 starting with '--'.
78 kwargs: Keyword arguments needed by the method parser.add_argument which adds
79 the contained argument to a parser.
80 """
81 self.names = names
82 self.kwargs = kwargs
85AblationArgument = ArgumentContainer(names=["--ablation"],
86 kwargs={"required": False,
87 "action": "store_true",
88 "help": "run ablation after configuration"})
89SelectorAblationArgument =\
90 ArgumentContainer(names=["--solver-ablation"],
91 kwargs={"required": False,
92 "action": "store_true",
93 "help": "construct a selector for "
94 "each solver ablation combination"})
96ActualMarginalContributionArgument = \
97 ArgumentContainer(names=["--actual"],
98 kwargs={"action": "store_true",
99 "help": "compute the marginal contribution "
100 "for the actual selector"})
102AlsoConstructSelectorAndReportArgument = \
103 ArgumentContainer(names=["--also-construct-selector-and-report"],
104 kwargs={"action": "store_true",
105 "help": "after running the solvers also construct the "
106 "selector and generate the report"})
108CleanupArgumentAll = \
109 ArgumentContainer(names=["--all"],
110 kwargs={"action": "store_true",
111 "help": "clean all output files"})
113CleanupArgumentRemove = \
114 ArgumentContainer(names=["--remove"],
115 kwargs={"action": "store_true",
116 "help": "remove all files in the platform, including "
117 "user data such as InstanceSets and Solvers"})
119ConfiguratorArgument = ArgumentContainer(names=["--configurator"],
120 kwargs={"type": Path,
121 "help": "path to configurator"})
123CPUTimeArgument = \
124 ArgumentContainer(names=["--cpu-time"],
125 kwargs={"type": int,
126 "help": "configuration budget per configurator run in "
127 "seconds (cpu)"})
129CutOffTimeArgument = \
130 ArgumentContainer(names=["--cutoff-time"],
131 kwargs={"type": int,
132 "help": "The duration the portfolio will run before the "
133 "solvers within the portfolio will be stopped "
134 "(default: "
135 f"{Settings.DEFAULT_general_target_cutoff_time})"})
137DeterministicArgument =\
138 ArgumentContainer(names=["--deterministic"],
139 kwargs={"action": "store_true",
140 "help": "Flag indicating the solver is deterministic"})
142DownloadExamplesArgument =\
143 ArgumentContainer(names=["--download-examples"],
144 kwargs={"action": argparse.BooleanOptionalAction,
145 "default": False,
146 "help": "Download the Examples into the directory."})
148ExtractorPathArgument = ArgumentContainer(names=["extractor_path"],
149 kwargs={"metavar": "extractor-path",
150 "type": str,
151 "help": "path or nickname of the "
152 "feature extractor"
153 })
155GenerateJSONArgument = ArgumentContainer(names=["--only-json"],
156 kwargs={"required": False,
157 "default": False,
158 "type": bool,
159 "help": "if set to True, only generate "
160 "machine readable output"
161 })
163InstancePathPositional = ArgumentContainer(names=["instance_path"],
164 kwargs={"type": Path,
165 "help": "Path to an instance (set)"})
167InstancePath = ArgumentContainer(names=["--instance-path"],
168 kwargs={"type": Path,
169 "help": "Path to an instance (set)"})
171InstanceSetTestArgument = \
172 ArgumentContainer(names=["--instance-set-test"],
173 kwargs={"required": False,
174 "type": Path,
175 "help": "path to test instance set (only for validating)"})
177InstanceSetTrainArgument = \
178 ArgumentContainer(names=["--instance-set-train"],
179 kwargs={"required": True,
180 "type": Path,
181 "help": "path to training instance set"})
183InstanceSetTestAblationArgument = \
184 ArgumentContainer(names=["--instance-set-test"],
185 kwargs={"required": False,
186 "type": str,
187 "help": "path to test instance set"})
189InstanceSetTrainAblationArgument = \
190 ArgumentContainer(names=["--instance-set-train"],
191 kwargs={"required": False,
192 "type": str,
193 "help": "path to training instance set"})
195InstanceSetTestReportArgument = \
196 ArgumentContainer(names=["--instance-set-test"],
197 kwargs={"required": False,
198 "type": str,
199 "help": "path to testing instance set included in "
200 "Sparkle for an algorithm configuration report"})
202InstanceSetTrainReportArgument = \
203 ArgumentContainer(names=["--instance-set-train"],
204 kwargs={"required": False,
205 "type": str,
206 "help": "path to training instance set included in "
207 "Sparkle for an algorithm configuration report"})
209InstancesPathArgument = ArgumentContainer(names=["instances_path"],
210 kwargs={"metavar": "instances-path",
211 "type": str,
212 "help": "path to the instance set"})
214InstancesPathRemoveArgument = \
215 ArgumentContainer(names=["instances_path"],
216 kwargs={"metavar": "instances-path",
217 "type": str,
218 "help": "path to or nickname of the instance set"})
220JobIDsArgument = ArgumentContainer(names=["--job-ids"],
221 kwargs={"required": False,
222 "nargs": "+",
223 "type": str,
224 "default": None,
225 "help": "job ID to wait for"})
227NicknameFeatureExtractorArgument = \
228 ArgumentContainer(names=["--nickname"],
229 kwargs={"type": str,
230 "help": "set a nickname for the feature extractor"})
232NicknameInstanceSetArgument = \
233 ArgumentContainer(names=["--nickname"],
234 kwargs={"type": str,
235 "help": "set a nickname for the instance set"})
237NicknamePortfolioArgument = \
238 ArgumentContainer(names=["--portfolio-name"],
239 kwargs={"type": Path,
240 "help": "Specify a name of the portfolio. "
241 "If none is given, one will be generated."})
243NicknameSolverArgument = \
244 ArgumentContainer(names=["--nickname"],
245 kwargs={"type": str,
246 "help": "set a nickname for the solver"})
248NoAblationReportArgument = ArgumentContainer(names=["--no-ablation"],
249 kwargs={"required": False,
250 "dest": "flag_ablation",
251 "default": True,
252 "const": False,
253 "nargs": "?",
254 "help": "turn off reporting on "
255 "ablation for an algorithm "
256 "configuration report"})
258NumberOfRunsConfigurationArgument = \
259 ArgumentContainer(names=["--number-of-runs"],
260 kwargs={"type": int,
261 "help": "number of configuration runs to execute"})
263NumberOfRunsAblationArgument = \
264 ArgumentContainer(names=["--number-of-runs"],
265 kwargs={"type": int,
266 "default": Settings.DEFAULT_config_number_of_runs,
267 "action": SetByUser,
268 "help": "Number of configuration runs to execute"})
270PerfectSelectorMarginalContributionArgument =\
271 ArgumentContainer(names=["--perfect"],
272 kwargs={"action": "store_true",
273 "help": "compute the marginal contribution "
274 "for the perfect selector"})
276RacingArgument = ArgumentContainer(names=["--racing"],
277 kwargs={"type": bool,
278 "default": Settings.
279 DEFAULT_ablation_racing,
280 "action": SetByUser,
281 "help": "Performs abaltion analysis with "
282 "racing"})
284RecomputeFeaturesArgument = \
285 ArgumentContainer(names=["--recompute"],
286 kwargs={"action": "store_true",
287 "help": "Re-run feature extractor for instances with "
288 "previously computed features"})
290RecomputeMarginalContributionArgument = \
291 ArgumentContainer(names=["--recompute"],
292 kwargs={"action": "store_true",
293 "help": "force marginal contribution to be recomputed even"
294 " when it already exists in file for the current "
295 "selector"})
297RecomputeMarginalContributionForSelectorArgument = \
298 ArgumentContainer(names=["--recompute-marginal-contribution"],
299 kwargs={"action": "store_true",
300 "help": "force marginal contribution to be recomputed even"
301 " when it already exists in file for the current "
302 "selector"})
304RecomputePortfolioSelectorArgument = \
305 ArgumentContainer(names=["--recompute-portfolio-selector"],
306 kwargs={"action": "store_true",
307 "help": "force the construction of a new portfolio "
308 "selector even when it already exists for the current "
309 "feature and performance data. NOTE: This will also "
310 "result in the computation of the marginal contributions "
311 "of solvers to the new portfolio selector."})
313RecomputeRunSolversArgument = \
314 ArgumentContainer(names=["--recompute"],
315 kwargs={"action": "store_true",
316 "help": "recompute the performance of all solvers on all "
317 "instances"})
319RunExtractorNowArgument = \
320 ArgumentContainer(names=["--run-extractor-now"],
321 kwargs={"default": False,
322 "action": "store_true",
323 "help": "immediately run the feature extractor(s) on all "
324 "the instances"})
326RunOnArgument = ArgumentContainer(names=["--run-on"],
327 kwargs={"type": Runner,
328 "choices": [Runner.LOCAL,
329 Runner.SLURM],
330 "action": EnumAction,
331 "help": "On which computer or cluster "
332 "environment to execute the "
333 "calculation."})
335RunSolverNowArgument = ArgumentContainer(names=["--run-solver-now"],
336 kwargs={"default": False,
337 "action": "store_true",
338 "help": "immediately run the solver(s) "
339 "on all instances"})
341SelectionReportArgument = \
342 ArgumentContainer(names=["--selection"],
343 kwargs={"action": "store_true",
344 "help": "set to generate a normal selection report"})
346SettingsFileArgument = \
347 ArgumentContainer(names=["--settings-file"],
348 kwargs={"type": Path,
349 "default": Settings.DEFAULT_settings_path,
350 "action": SetByUser,
351 "help": "Specify the settings file to use in case you want"
352 " to use one other than the default"})
354SkipChecksArgument = ArgumentContainer(
355 names=["--skip-checks"],
356 kwargs={"dest": "run_checks",
357 "default": True,
358 "action": "store_false",
359 "help": "Checks the solver's functionality by testing it on an instance "
360 "and the pcs file, when applicable."})
362SnapshotArgument = ArgumentContainer(names=["snapshot_file_path"],
363 kwargs={"metavar": "snapshot-file-path",
364 "type": str,
365 "help": "path to the snapshot file"})
367SolverArgument = ArgumentContainer(names=["--solver"],
368 kwargs={"required": True,
369 "type": Path,
370 "help": "path to solver"})
372SolversArgument = ArgumentContainer(names=["--solvers"],
373 kwargs={"required": False,
374 "nargs": "+",
375 "type": list[str],
376 "help": "Specify the list of solvers to be "
377 "used. If not specifed, all solvers "
378 "known in Sparkle will be used."})
380SolverCallsArgument = \
381 ArgumentContainer(names=["--solver-calls"],
382 kwargs={"type": int,
383 "help": "number of solver calls to execute"})
385SolverSeedsArgument = \
386 ArgumentContainer(names=["--solver-seeds"],
387 kwargs={"type": int,
388 "help": "number of random seeds per solver to execute"})
390SolverRemoveArgument = \
391 ArgumentContainer(names=["solver"],
392 kwargs={"metavar": "solver",
393 "type": str,
394 "help": "name, path to or nickname of the solver"})
396SolverPathArgument = ArgumentContainer(names=["solver_path"],
397 kwargs={"metavar": "solver-path",
398 "type": str,
399 "help": "path to the solver"})
401SolverReportArgument = ArgumentContainer(names=["--solver"],
402 kwargs={"required": False,
403 "type": str,
404 "default": None,
405 "help": "path to solver for an "
406 "algorithm configuration report"})
408TargetCutOffTimeAblationArgument = \
409 ArgumentContainer(names=["--target-cutoff-time"],
410 kwargs={"type": int,
411 "default": Settings.DEFAULT_general_target_cutoff_time,
412 "action": SetByUser,
413 "help": "cutoff time per target algorithm run in seconds"})
415TargetCutOffTimeConfigurationArgument = \
416 ArgumentContainer(names=["--target-cutoff-time"],
417 kwargs={"type": int,
418 "help": "cutoff time per target algorithm run in seconds"})
420TargetCutOffTimeRunSolversArgument = \
421 ArgumentContainer(names=["--target-cutoff-time"],
422 kwargs={"type": int,
423 "help": "cutoff time per target algorithm run in seconds"})
425TargetCutOffTimeValidationArgument = \
426 ArgumentContainer(names=["--target-cutoff-time"],
427 kwargs={"type": int,
428 "default": Settings.DEFAULT_general_target_cutoff_time,
429 "action": SetByUser,
430 "help": "cutoff time per target algorithm run in seconds"})
432TestCaseDirectoryArgument = \
433 ArgumentContainer(names=["--test-case-directory"],
434 kwargs={"type": str,
435 "default": None,
436 "help": "Path to test case directory of an instance set "
437 + "for a selection report"})
439UseFeaturesArgument = ArgumentContainer(names=["--use-features"],
440 kwargs={"required": False,
441 "action": "store_true",
442 "help": "use the training set's features"
443 " for configuration"})
445ValidateArgument = ArgumentContainer(names=["--validate"],
446 kwargs={"required": False,
447 "action": "store_true",
448 "help": "validate after configuration"})
450VerboseArgument = ArgumentContainer(names=["--verbose", "-v"],
451 kwargs={"action": "store_true",
452 "help": "output status in verbose mode"})
454WallClockTimeArgument = \
455 ArgumentContainer(names=["--wallclock-time"],
456 kwargs={"type": int,
457 "help": "configuration budget per configurator run in "
458 "seconds (wallclock)"})
460SelectorTimeoutArgument = \
461 ArgumentContainer(names=["--selector-timeout"],
462 kwargs={"type": int,
463 "default": Settings.DEFAULT_portfolio_construction_timeout,
464 "help": "Cuttoff time (in seconds) for the algorithm"
465 "selector construction"})
467SparkleObjectiveArgument = \
468 ArgumentContainer(names=["--objectives"],
469 kwargs={"type": str,
470 "help": "the comma seperated objective(s) to use."})