Coverage for sparkle/platform/generate_report_for_configuration.py: 85%
177 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-09-27 09:10 +0000
1#!/usr/bin/env python3
2# -*- coding: UTF-8 -*-
3"""Helper functions for algorithm configuration report generation."""
4from __future__ import annotations
6import sys
7from pathlib import Path
9from scipy.stats import linregress
11from sparkle.platform import latex as stex
12from sparkle.solver.ablation import AblationScenario
13from sparkle.solver.validator import Validator
14from sparkle.configurator.configurator import Configurator, ConfigurationScenario
15from sparkle.solver import Solver
16from sparkle.instance import InstanceSet
17from sparkle.configurator.implementations import SMAC2
18from sparkle.types import SparkleObjective
19from sparkle import about
22def get_features_bool(configurator_scenario: ConfigurationScenario,
23 solver_name: str, train_set: InstanceSet) -> str:
24 """Return a bool string for latex indicating whether features were used.
26 True if a feature file is given in the scenario file, false otherwise.
28 Args:
29 solver_name: Name of the solver
30 instance_set_train_name: Name of the instance set used for training
32 Returns:
33 A string describing whether features are used
34 """
35 scenario_file = configurator_scenario.directory \
36 / f"{solver_name}_{train_set.name}_scenario.txt"
38 for line in scenario_file.open("r").readlines():
39 if line.split(" ")[0] == "feature_file":
40 return "\\featurestrue"
41 return "\\featuresfalse"
44def get_average_performance(results: list[list[str]],
45 objective: SparkleObjective) -> float:
46 """Return the PAR score for a given results file and cutoff time.
48 Args:
49 results_file: Name of the result file
50 objective: The objective to average
52 Returns:
53 Average performance value
54 """
55 instance_per_dict = get_dict_instance_to_performance(results,
56 objective)
57 num_instances = len(instance_per_dict.keys())
58 sum_par = sum(float(instance_per_dict[instance]) for instance in instance_per_dict)
59 return float(sum_par / num_instances)
62def get_dict_instance_to_performance(results: list[list[str]],
63 objective: SparkleObjective) -> dict[str, float]:
64 """Return a dictionary of instance names and their performance.
66 Args:
67 results: Results from CSV
68 objective: The Sparkle Objective we are converting for
69 Returns:
70 A dictionary containing the performance for each instance
71 """
72 value_column = results[0].index(objective.name)
73 results_per_instance = {}
74 for row in results[1:]:
75 value = float(row[value_column])
76 results_per_instance[Path(row[3]).name] = value
77 return results_per_instance
80def get_ablation_bool(scenario: AblationScenario) -> str:
81 """Return the ablation bool as LaTeX string.
83 Args:
84 solver: The solver object
85 instance_train_name: Name of the trianing instance set
86 instance_test_name: Name of the testing instance set
88 Returns:
89 A string describing whether ablation was run or not
90 """
91 if scenario.check_for_ablation():
92 return "\\ablationtrue"
93 return "\\ablationfalse"
96def get_data_for_plot(configured_results: list[list[str]],
97 default_results: list[list[str]],
98 objective: SparkleObjective) -> list:
99 """Return the required data to plot.
101 Creates a nested list of performance values algorithm runs with default and
102 configured parameters on instances in a given instance set.
104 Args:
105 configured_results_dir: Directory of results for configured solver
106 default_results_dir: Directory of results for default solver
107 run_cutoff_time: Cutoff time
109 Returns:
110 A list of lists containing data points
111 """
112 dict_instance_to_par_default = get_dict_instance_to_performance(
113 default_results, objective)
114 dict_instance_to_par_configured = get_dict_instance_to_performance(
115 configured_results, objective)
117 instances = (dict_instance_to_par_default.keys()
118 & dict_instance_to_par_configured.keys())
119 if (len(dict_instance_to_par_default) != len(instances)):
120 print("""ERROR: Number of instances does not match
121 the number of performance values for the default configuration.""")
122 sys.exit(-1)
123 points = []
124 for instance in instances:
125 point = [dict_instance_to_par_default[instance],
126 dict_instance_to_par_configured[instance]]
127 points.append(point)
129 return points
132def get_figure_configure_vs_default(configured_results: list[list[str]],
133 default_results: list[list[str]],
134 target_directory: Path,
135 figure_filename: str,
136 performance_measure: str,
137 run_cutoff_time: float,
138 objective: SparkleObjective) -> str:
139 """Create a figure comparing the configured and default solver.
141 Base function to create a comparison plot of a given instance set between the default
142 and configured performance.
144 Args:
145 configured_results_dir: Directory of results for configured solver
146 default_results_dir: Directory of results for default solver
147 target_directory: Directory for the configuration reports
148 figure_filename: Filename for the figure
149 run_cutoff_time: Cutoff time
151 Returns:
152 A string containing the latex command to include the figure
153 """
154 points = get_data_for_plot(configured_results, default_results,
155 objective)
157 plot_params = {"xlabel": f"Default parameters [{performance_measure}]",
158 "ylabel": f"Configured parameters [{performance_measure}]",
159 "scale": "linear",
160 "limit_min": 1.5,
161 "limit_max": 1.5,
162 "replace_zeros": False,
163 "output_dir": target_directory
164 }
165 # Check if the scale of the axis can be considered linear
166 linearity_x = linregress([p[0] for p in points], range(len(points))).rvalue > 0.5
167 linearity_y = linregress([p[1] for p in points], range(len(points))).rvalue > 0.5
168 if not linearity_x or not linearity_y:
169 plot_params["scale"] = "log"
170 plot_params["replace_zeros"] = True
172 stex.generate_comparison_plot(points,
173 figure_filename,
174 **plot_params)
176 return f"\\includegraphics[width=0.6\\textwidth]{{{figure_filename}}}"
179def get_figure_configured_vs_default_on_instance_set(solver: Solver,
180 instance_set_name: str,
181 res_default: list[list[str]],
182 res_conf: list[list[str]],
183 target_directory: Path,
184 smac_objective: str,
185 run_cutoff_time: float,
186 objective: SparkleObjective,
187 data_type: str = "train") -> str:
188 """Create a figure comparing the configured and default solver on the training set.
190 Manages the creation of a comparison plot of the instances in the train instance set
191 for the report by gathering the proper files and choosing the plotting parameters
192 based on the performance measure.
194 Args:
195 solver: The solver object
196 instance_set_train_name: Name of the instance set for training
197 configuration_reports_directory: Directory to the configuration reports
198 run_cutoff_time: Cutoff time
200 Returns:
201 A string containing the latex comand to include the figure
202 """
203 data_plot_configured_vs_default_on_instance_set_filename = (
204 f"data_{solver.name}_configured_vs_default_on_{instance_set_name}_{data_type}")
205 return get_figure_configure_vs_default(
206 res_conf, res_default, target_directory,
207 data_plot_configured_vs_default_on_instance_set_filename,
208 smac_objective,
209 run_cutoff_time,
210 objective)
213def get_timeouts_instanceset(solver: Solver,
214 instance_set: InstanceSet,
215 configurator: Configurator,
216 validator: Validator,
217 cutoff: float) -> tuple[int, int, int]:
218 """Return the number of timeouts by configured, default and both on the testing set.
220 Args:
221 solver: The solver object
222 instance_set: Instance Set
223 configurator: Configurator
224 validator: Validator
225 cutoff: Cutoff time
227 Returns:
228 A tuple containing the number of timeouts for the different configurations
229 """
230 objective = configurator.scenario.sparkle_objective
231 _, config = configurator.get_optimal_configuration(
232 solver, instance_set, objective)
233 res_default = validator.get_validation_results(solver,
234 instance_set,
235 config="")
236 res_conf = validator.get_validation_results(solver,
237 instance_set,
238 config=config)
239 dict_instance_to_par_configured = get_dict_instance_to_performance(
240 res_conf, objective)
241 dict_instance_to_par_default = get_dict_instance_to_performance(
242 res_default, objective)
244 return get_timeouts(dict_instance_to_par_configured,
245 dict_instance_to_par_default, cutoff)
248def get_timeouts(instance_to_par_configured: dict,
249 instance_to_par_default: dict,
250 cutoff: float) -> tuple[int, int, int]:
251 """Return the number of timeouts for given dicts.
253 Args:
254 dict_instance_to_par_configured: _description_
255 dict_instance_to_par_default: _description_
256 cutoff: Cutoff value
258 Returns:
259 A tuple containing timeout values
260 """
261 configured_timeouts = 0
262 default_timeouts = 0
263 overlapping_timeouts = 0
265 for instance in instance_to_par_configured:
266 configured_par = instance_to_par_configured[instance]
267 default_par = instance_to_par_default[instance]
268 # Count the amount of values that are equal to timeout
269 configured_timeouts += (configured_par > cutoff)
270 default_timeouts += (default_par > cutoff)
271 overlapping_timeouts += (configured_par > cutoff
272 and default_par > cutoff)
274 return configured_timeouts, default_timeouts, overlapping_timeouts
277def get_ablation_table(scenario: AblationScenario) -> str:
278 """Generate a LaTeX table of the ablation path.
280 This is the result of the ablation analysis to determine the parameter importance.
282 Args:
283 solver: The solver object
284 instance_set_train_name: Name of the instance set for training
285 instance_set_test_name: Name of the instance set for testing
287 Returns:
288 A string containing the LaTeX table code of the ablation path
289 """
290 results = scenario.read_ablation_table()
291 table_string = r"\begin{tabular}{rp{0.25\linewidth}rrr}"
292 # "Round", "Flipped parameter", "Source value", "Target value", "Validation result"
293 for i, line in enumerate(results):
294 # If this fails something has changed in the representation of ablation tables
295 if len(line) != 5:
296 print("""ERROR: something has changed with the representation
297 of ablation tables""")
298 sys.exit(-1)
299 if i == 0:
300 line = [f"\\textbf{{{word}}}" for word in line]
302 # Put multiple variable changes in one round on a seperate line
303 if (len(line[1].split(",")) > 1
304 and len(line[1].split(",")) == len(line[2].split(","))
305 and len(line[1].split(",")) == len(line[3].split(","))):
306 params = line[1].split(",")
307 default_values = line[2].split(",")
308 flipped_values = line[3].split(",")
310 sublines = len(params)
311 for subline in range(sublines):
312 round = "" if subline != 0 else line[0]
313 result = "" if subline + 1 != sublines else line[-1]
314 printline = [round, params[subline], default_values[subline],
315 flipped_values[subline], result]
316 table_string += " & ".join(printline) + " \\\\ "
317 else:
318 table_string += " & ".join(line) + " \\\\ "
319 if i == 0:
320 table_string += "\\hline "
321 table_string += "\\end{tabular}"
323 return table_string
326def configuration_report_variables(target_dir: Path,
327 solver: Solver,
328 configurator: Configurator,
329 validator: Validator,
330 extractor_dir: Path,
331 bib_path: Path,
332 instance_set_train: InstanceSet,
333 extractor_cuttoff: int,
334 instance_set_test: InstanceSet = None,
335 ablation: AblationScenario = None) -> dict:
336 """Return a dict matching LaTeX variables and their values.
338 Args:
339 solver: Object representation of the Solver
340 instance_set_train: Path of the instance set for training
341 instance_set_test: Path of the instance set for testing. Defaults to None.
342 ablation: Whether or not ablation is used. Defaults to True.
344 Returns:
345 A dictionary containing the variables and values
346 """
347 has_test = instance_set_test is not None
349 full_dict = get_dict_variable_to_value_common(solver,
350 configurator,
351 validator,
352 ablation,
353 bib_path,
354 instance_set_train,
355 target_dir)
357 if has_test:
358 test_dict = get_dict_variable_to_value_test(target_dir,
359 solver,
360 configurator,
361 validator,
362 ablation,
363 instance_set_train,
364 instance_set_test)
365 full_dict.update(test_dict)
366 full_dict["testBool"] = f"\\test{str(has_test).lower()}"
368 if ablation is None:
369 full_dict["ablationBool"] = "\\ablationfalse"
371 if full_dict["featuresBool"] == "\\featurestrue":
372 full_dict["numFeatureExtractors"] =\
373 len([p for p in extractor_dir.iterdir()])
374 full_dict["featureExtractorList"] =\
375 stex.list_to_latex([(p.name, "") for p in extractor_dir.iterdir()])
376 full_dict["featureComputationCutoffTime"] = extractor_cuttoff
378 return full_dict
381def get_dict_variable_to_value_common(solver: Solver,
382 configurator: Configurator,
383 validator: Validator,
384 ablation: AblationScenario,
385 bibliography_path: Path,
386 train_set: InstanceSet,
387 target_directory: Path) -> dict:
388 """Return a dict matching LaTeX variables and values used for all config. reports.
390 Args:
391 Solver: The solver object
392 instance_set_train: Path of the instance set for training
393 instance_set_test: Path of the instance set for testing
394 target_directory: Path to directory with configuration reports
396 Returns:
397 A dictionary containing the variables and values
398 """
399 objective = configurator.scenario.sparkle_objective
400 _, opt_config = configurator.get_optimal_configuration(
401 solver, train_set, objective)
402 res_default = validator.get_validation_results(
403 solver, train_set, config="")
404 res_conf = validator.get_validation_results(
405 solver, train_set, config=opt_config)
406 instance_names = set([res[3] for res in res_default])
407 opt_config_list = [f"{key}: {value}" for key, value in
408 Solver.config_str_to_dict(opt_config).items()]
410 latex_dict = {"bibliographypath": bibliography_path.absolute()}
411 latex_dict["performanceMeasure"] = objective.name
412 smac_run_obj = SMAC2.get_smac_run_obj(objective)
414 if smac_run_obj == "RUNTIME":
415 latex_dict["runtimeBool"] = "\\runtimetrue"
416 elif smac_run_obj == "QUALITY":
417 latex_dict["runtimeBool"] = "\\runtimefalse"
419 latex_dict["solver"] = solver.name
420 latex_dict["instanceSetTrain"] = train_set.name
421 latex_dict["sparkleVersion"] = about.version
422 latex_dict["numInstanceInTrainingInstanceSet"] = len(instance_names)
424 run_cutoff_time = configurator.scenario.cutoff_time
425 latex_dict["numSmacRuns"] = configurator.scenario.number_of_runs
426 latex_dict["smacObjective"] = smac_run_obj
427 latex_dict["smacWholeTimeBudget"] = configurator.scenario.wallclock_time
428 latex_dict["smacEachRunCutoffTime"] = run_cutoff_time
429 latex_dict["optimisedConfiguration"] = stex.list_to_latex(opt_config_list)
430 latex_dict["optimisedConfigurationTrainingPerformancePAR"] =\
431 get_average_performance(res_conf, objective)
432 latex_dict["defaultConfigurationTrainingPerformancePAR"] =\
433 get_average_performance(res_default, objective)
435 str_value = get_figure_configured_vs_default_on_instance_set(
436 solver, train_set.name, res_default, res_conf, target_directory,
437 smac_run_obj, float(run_cutoff_time), objective)
438 latex_dict["figure-configured-vs-default-train"] = str_value
440 # Retrieve timeout numbers for the training instances
441 configured_timeouts_train, default_timeouts_train, overlapping_timeouts_train =\
442 get_timeouts_instanceset(solver, train_set, configurator, validator,
443 run_cutoff_time)
445 latex_dict["timeoutsTrainDefault"] = default_timeouts_train
446 latex_dict["timeoutsTrainConfigured"] = configured_timeouts_train
447 latex_dict["timeoutsTrainOverlap"] = overlapping_timeouts_train
448 latex_dict["ablationBool"] = get_ablation_bool(ablation)
449 latex_dict["ablationPath"] = get_ablation_table(ablation)
450 latex_dict["featuresBool"] = get_features_bool(
451 configurator.scenario, solver.name, train_set)
453 return latex_dict
456def get_dict_variable_to_value_test(target_dir: Path,
457 solver: Solver,
458 configurator: Configurator,
459 validator: Validator,
460 ablation: AblationScenario,
461 train_set: InstanceSet,
462 test_set: InstanceSet) -> dict:
463 """Return a dict matching test set specific latex variables with their values.
465 Args:
466 target_dir: Path to where output should go
467 solver: The solver object
468 configurator: Configurator for which the report is generated
469 validator: Validator that provided the data set results
470 train_set: Instance set for training
471 test_set: Instance set for testing
473 Returns:
474 A dictionary containting the variables and their values
475 """
476 _, config = configurator.get_optimal_configuration(
477 solver, train_set, configurator.scenario.sparkle_objective)
478 res_default = validator.get_validation_results(
479 solver, test_set, config="")
480 res_conf = validator.get_validation_results(
481 solver, test_set, config=config)
482 instance_names = set([res[3] for res in res_default])
483 run_cutoff_time = configurator.scenario.cutoff_time
484 objective = configurator.scenario.sparkle_objective
485 test_dict = {"instanceSetTest": test_set.name}
486 test_dict["numInstanceInTestingInstanceSet"] = len(instance_names)
487 test_dict["optimisedConfigurationTestingPerformancePAR"] =\
488 get_average_performance(res_conf, objective)
489 test_dict["defaultConfigurationTestingPerformancePAR"] =\
490 get_average_performance(res_default, objective)
491 smac_run_obj = SMAC2.get_smac_run_obj(
492 configurator.scenario.sparkle_objective)
493 test_dict["figure-configured-vs-default-test"] =\
494 get_figure_configured_vs_default_on_instance_set(
495 solver, test_set.name, res_default, res_conf, target_dir, smac_run_obj,
496 float(run_cutoff_time),
497 configurator.scenario.sparkle_objective, data_type="test")
499 # Retrieve timeout numbers for the testing instances
500 configured_timeouts_test, default_timeouts_test, overlapping_timeouts_test =\
501 get_timeouts_instanceset(solver,
502 test_set,
503 configurator,
504 validator,
505 run_cutoff_time)
507 test_dict["timeoutsTestDefault"] = default_timeouts_test
508 test_dict["timeoutsTestConfigured"] = configured_timeouts_test
509 test_dict["timeoutsTestOverlap"] = overlapping_timeouts_test
510 test_dict["ablationBool"] = get_ablation_bool(ablation)
511 test_dict["ablationPath"] = get_ablation_table(ablation)
512 return test_dict
515def generate_report_for_configuration(solver: Solver,
516 configurator: Configurator,
517 validator: Validator,
518 extractor_dir: Path,
519 target_path: Path,
520 latex_template_path: Path,
521 bibliography_path: Path,
522 train_set: InstanceSet,
523 extractor_cuttoff: int,
524 test_set: InstanceSet = None,
525 ablation: AblationScenario = None) -> None:
526 """Generate a report for algorithm configuration.
528 Args:
529 solver: Object representation of the solver
530 configurator: Configurator for the report
531 validator: Validator that validated the configurator
532 extractor_dir: Path to the extractor used
533 target_path: Where the report files will be placed.
534 latex_template_path: Path to the template to use for the report
535 bibliography_path: The bib corresponding to the latex template
536 train_set: Instance set for training
537 extractor_cuttoff: Cut off for extractor
538 test_set: Instance set for testing
539 ablation: Whether or not ablation is used. Defaults to True.
540 """
541 target_path.mkdir(parents=True, exist_ok=True)
542 variables_dict = configuration_report_variables(
543 target_path, solver, configurator, validator, extractor_dir, bibliography_path,
544 train_set, extractor_cuttoff, test_set,
545 ablation)
546 stex.generate_report(latex_template_path,
547 "template-Sparkle-for-configuration.tex",
548 target_path,
549 "Sparkle_Report_for_Configuration",
550 variables_dict)