Coverage for sparkle/CLI/generate_report.py: 24%
382 statements
« prev ^ index » next coverage.py v7.9.1, created at 2025-07-01 13:21 +0000
1#!/usr/bin/env python3
2"""Sparkle command to generate a report for an executed experiment."""
3import sys
4import shutil
5import argparse
6from pathlib import Path
7import time
8import json
10import pylatex as pl
11from sparkle import __version__ as __sparkle_version__
13from sparkle.CLI.help import global_variables as gv
14from sparkle.CLI.help import resolve_object_name
15from sparkle.CLI.help import logging as sl
16from sparkle.CLI.help import argparse_custom as ac
18from sparkle.solver import Solver
19from sparkle.instance import Instance_Set
20from sparkle.selector import Extractor
21from sparkle.structures import PerformanceDataFrame, FeatureDataFrame
22from sparkle.configurator.configurator import ConfigurationScenario
23from sparkle.selector.selector import SelectionScenario
24from sparkle.types import SolverStatus
26from sparkle.platform import latex
27from sparkle.platform.output.configuration_output import ConfigurationOutput
28from sparkle.platform.output.selection_output import SelectionOutput
31MAX_DEC = 4 # Maximum decimals used for each reported value
34def parser_function() -> argparse.ArgumentParser:
35 """Define the command line arguments."""
36 parser = argparse.ArgumentParser(
37 description="Generates a report for all known selection, configuration and "
38 "parallel portfolio scenarios will be generated.",
39 epilog="If you wish to filter specific solvers, instance sets, ... have a look "
40 "at the command line arguments.")
41 # Add argument for filtering solvers
42 parser.add_argument(*ac.SolversReportArgument.names,
43 **ac.SolversReportArgument.kwargs)
44 # Add argument for filtering instance sets
45 parser.add_argument(*ac.InstanceSetsReportArgument.names,
46 **ac.InstanceSetsReportArgument.kwargs)
47 # Add argument for filtering configurators?
48 # Add argument for filtering selectors?
49 # Add argument for filtering ??? scenario ids? configuration ids?
50 parser.add_argument(*ac.GenerateJSONArgument.names,
51 **ac.GenerateJSONArgument.kwargs)
52 return parser
55def generate_configuration_section(report: pl.Document, scenario: ConfigurationScenario,
56 scenario_output: ConfigurationOutput) -> None:
57 """Generate a section for a configuration scenario."""
58 report_dir = Path(report.default_filepath).parent
59 time_stamp = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))
60 plot_dir = report_dir /\
61 f"{scenario.configurator.__name__}_{scenario.name}_plots_{time_stamp}"
62 plot_dir.mkdir(exist_ok=True)
64 # 1. Write section intro
65 report.append(pl.Section(
66 f"{scenario.configurator.__name__} Configuration: "
67 f"{scenario.solver.name} on {scenario.instance_set.name}"))
68 report.append("In this scenario, ")
69 report.append(pl.UnsafeCommand(
70 f"textbf{{{scenario.configurator.__name__}}} "
71 f"({scenario.configurator.full_name})~\\cite"
72 f"{{{scenario.configurator.__name__}}} with version "
73 f"{scenario.configurator.version} was used for configuration. "))
74 report.append(
75 f"The Solver {scenario.solver} was optimised on training set "
76 f"{scenario.instance_set}. The scenario was run {scenario.number_of_runs} "
77 f"times independently with different seeds, yielding {scenario.number_of_runs} "
78 f"configurations. The cutoff time for the solver was set to "
79 f"{scenario.solver_cutoff_time} seconds. The optimised objective is "
80 f"{scenario.sparkle_objectives[0]}. Each Configuration was evaluated on the "
81 "training set to determine the best configuration, e.g. the best "
82 f"{scenario.sparkle_objectives[0]} value on the training set.")
84 # 2. Report all the configurator settings in table format
85 report.append(pl.Subsection("Configurator Settings"))
86 report.append("The following settings were used for "
87 f"{scenario.configurator.__name__}:\n")
88 tabular = pl.Tabular("l|r")
89 tabular.add_row("Setting", "Value")
90 tabular.add_hline()
91 for setting, value in scenario.serialise().items():
92 tabular.add_row([setting, str(value)])
93 table_conf_settings = pl.Table(position="h")
94 table_conf_settings.append(pl.UnsafeCommand("centering"))
95 table_conf_settings.append(tabular)
96 table_conf_settings.add_caption("Configurator Settings")
97 report.append(table_conf_settings)
99 # 3. Report details on instance and solver used
100 report.append(pl.Subsection("Solver & Instance Set(s) Details"))
101 cs = scenario_output.solver.get_configuration_space()
102 report.append(f"The solver {scenario_output.solver} was configured using "
103 f"{len(cs.values())} configurable (hyper)parameters. "
104 f"The configuration space has {len(cs.conditions)} conditions. ")
105 report.append("The following instance sets were used for the scenario:")
106 with report.create(pl.Itemize()) as instance_set_latex_list:
107 for instance_set in [
108 scenario_output.instance_set_train] + scenario_output.test_instance_sets:
109 training_set_name = instance_set.name.replace("_", " ") # Latex fix
110 instance_set_latex_list.add_item(
111 pl.UnsafeCommand(
112 f"textbf{{{training_set_name}}} ({instance_set.size} instances)"))
114 # Function to generate a results summary of default vs best on an instance set
115 def instance_set_summary(instance_set_name: str) -> None:
116 """Generate a results summary of default vs best on an instance set."""
117 instance_set_results = scenario_output.instance_set_results[instance_set_name]
118 report.append(f"The {scenario.sparkle_objectives[0]} value of the Default "
119 f"Configuration on {instance_set_name} was ")
120 report.append(pl.UnsafeCommand(
121 f"textbf{{{round(instance_set_results.default_performance, MAX_DEC)}}}.\n"))
122 report.append(f"The {scenario.sparkle_objectives[0]} value of the Best "
123 f"Configuration on {instance_set_name} was ")
124 report.append(pl.UnsafeCommand(
125 f"textbf{{{round(instance_set_results.best_performance, MAX_DEC)}}}.\n"))
126 report.append("In ")
127 report.append(latex.AutoRef(f"fig:bestvsdefault{instance_set_name}{time_stamp}"))
128 report.append(pl.utils.bold(" ")) # Force white space
129 report.append("the results are plotted per instance.")
130 # Create graph to compare best configuration vs default on the instance set
131 import pandas as pd
132 df = pd.DataFrame(
133 [instance_set_results.default_instance_performance,
134 instance_set_results.best_instance_performance],
135 index=["Default Configuration", "Best Configuration"], dtype=float).T
136 plot = latex.comparison_plot(df, None)
137 plot_path = plot_dir /\
138 f"{scenario_output.best_configuration_key}_vs_"\
139 f"Default_{instance_set_name}.pdf"
140 plot.write_image(plot_path, width=500, height=500)
141 with report.create(pl.Figure(position="h")) as figure:
142 figure.add_image(str(plot_path.relative_to(report_dir)),
143 width=pl.utils.NoEscape(r"0.6\textwidth"))
144 figure.add_caption(
145 f"Best vs Default Performance on {instance_set_name} "
146 f"({scenario.sparkle_objectives[0]})")
147 figure.append(pl.UnsafeCommand(
148 r"label{"
149 f"fig:bestvsdefault{instance_set_name}{time_stamp}"
150 r"}"))
151 if scenario.sparkle_objectives[0].time: # Write status table
152 report.append("The following Solver status were found per instance:")
153 tabular = pl.Tabular("l|c|c|c")
154 tabular.add_row("Status", "Default", "Best", "Overlap")
155 tabular.add_hline()
156 # Count the statuses
157 for status in SolverStatus:
158 default_count, best_count, overlap_count = 0, 0, 0
159 for instance in instance_set_results.instance_status_default.keys():
160 instance = str(instance)
161 default_hit = instance_set_results.instance_status_default[
162 instance] == status
163 best_hit = instance_set_results.instance_status_best[
164 instance] == status
165 default_count += default_hit
166 best_count += best_hit
167 overlap_count += (default_hit and best_hit)
168 if default_count or best_count:
169 tabular.add_row(
170 status, default_count, best_count, overlap_count
171 )
172 table_status_values = pl.Table(position="h")
173 table_status_values.append(pl.UnsafeCommand("centering"))
174 table_status_values.append(tabular)
175 table_status_values.add_caption(
176 "Status count for the best and default configuration.")
177 report.append(table_status_values)
179 # 4. Report the results of the best configuration on the training set vs the default
180 report.append(pl.Subsection(
181 f"Comparison of Default and Best Configuration on Training Set "
182 f"{scenario_output.instance_set_train.name}"))
183 instance_set_summary(scenario_output.instance_set_train.name)
185 # 5. Report the actual config values
186 report.append(pl.Subsubsection("Best Configuration Values"))
187 if scenario_output.best_configuration_key ==\
188 PerformanceDataFrame.default_configuration:
189 report.append("The configurator failed to find a better configuration than the "
190 "default configuration on the training set in this scenario.")
191 else:
192 report.append("The following parameter values "
193 "were found to be the best on the training set:\n")
194 tabular = pl.Tabular("l|r")
195 tabular.add_row("Parameter", "Value")
196 tabular.add_hline()
197 for parameter, value in scenario_output.best_configuration.items():
198 tabular.add_row([parameter, str(value)])
199 table_best_values = pl.Table(position="h")
200 table_best_values.append(pl.UnsafeCommand("centering"))
201 table_best_values.append(tabular)
202 table_best_values.add_caption("Best found configuration values")
203 report.append(table_best_values)
205 # 6. Report the results of best vs default conf on the test sets
207 for test_set in scenario_output.test_instance_sets:
208 report.append(pl.Subsection(
209 f"Comparison of Default and Best Configuration on Test Set "
210 f"{test_set.name}"))
211 instance_set_summary(test_set.name)
213 # 7. Report the parameter ablation scenario if present
214 if scenario.ablation_scenario:
215 report.append(pl.Subsection("Parameter importance via Ablation"))
216 report.append("Ablation analysis ")
217 report.append(pl.UnsafeCommand(r"cite{FawcettHoos16} "))
218 test_set = scenario.ablation_scenario.test_set
219 if not scenario.ablation_scenario.test_set:
220 test_set = scenario.ablation_scenario.train_set
221 report.append(
222 f"is performed from the default configuration of {scenario.solver} to the "
223 f"best found configuration ({scenario_output.best_configuration_key}) "
224 "to see which parameter changes between them contribute most to the improved"
225 " performance. The ablation path uses the training set "
226 f"{scenario.ablation_scenario.train_set.name} and validation is performed "
227 f"on the test set {test_set.name}. The set of parameters that differ in the "
228 "two configurations will form the ablation path. Starting from the default "
229 "configuration, the path is computed by performing a sequence of rounds. In"
230 " a round, each available parameter is flipped in the configuration and is "
231 "validated on its performance. The flipped parameter with the best "
232 "performance in that round, is added to the configuration and the next round"
233 " starts with the remaining parameters. This repeats until all parameters "
234 "are flipped, which is the best found configuration. The analysis resulted "
235 "in the ablation presented in ")
236 report.append(latex.AutoRef("tab:ablationtable"))
237 report.append(".")
239 # Add ablation table
240 tabular = pl.Tabular("r|l|r|r|r")
241 data = scenario.ablation_scenario.read_ablation_table()
242 for index, row in enumerate(data):
243 tabular.add_row(*row)
244 if index == 0:
245 tabular.add_hline()
246 table_ablation = pl.Table(position="h")
247 table_ablation.append(pl.UnsafeCommand("centering"))
248 table_ablation.append(tabular)
249 table_ablation.add_caption("Ablation table")
250 table_ablation.append(pl.UnsafeCommand(r"label{tab:ablationtable}"))
251 report.append(table_ablation)
254def generate_selection_section(report: pl.Document, scenario: SelectionScenario,
255 scenario_output: SelectionOutput) -> None:
256 """Generate a section for a selection scenario."""
257 report_dir = Path(report.default_filepath).parent
258 time_stamp = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))
259 plot_dir = report_dir / f"{scenario.name.replace(' ', '_')}_plots_{time_stamp}"
260 plot_dir.mkdir(exist_ok=True)
261 report.append(pl.Section(
262 f"Selection: {scenario.selector.model_class.__name__} on "
263 f"{' '.join([s[0] for s in scenario_output.training_instance_sets])}"))
264 report.append(f"In this scenario, a {scenario.selector.model_class.__name__} "
265 f" ({scenario.selector.selector_class.__name__}) was trained on the "
266 "performance and feature data using ASF-lib. The following solvers "
267 f"were run with a cutoff time of {scenario.solver_cutoff} seconds:")
268 with report.create(pl.Itemize()) as solver_latex_list:
269 for solver_name in scenario_output.solvers.keys():
270 solver_name = solver_name.replace("_", " ")
271 solver_latex_list.add_item(
272 pl.UnsafeCommand(
273 f"textbf{{{solver_name}}} "
274 f"({len(scenario_output.solvers[solver_name])} configurations)"))
275 # Report training instance sets
276 report.append("The following training instance sets were used:")
277 with report.create(pl.Itemize()) as instance_set_latex_list:
278 for training_set_name, set_size in scenario_output.training_instance_sets:
279 training_set_name = training_set_name.replace("_", " ") # Latex fix
280 instance_set_latex_list.add_item(
281 pl.UnsafeCommand(
282 f"textbf{{{training_set_name}}} "
283 f"({set_size} instances)"))
284 # Report feature extractors
285 report.append("The following feature extractors were used with a extractor cutoff "
286 f"time of {scenario.extractor_cutoff} seconds:")
287 with report.create(pl.Itemize()) as feature_extractor_latex_list:
288 for feature_extractor_name in scenario.feature_extractors:
289 extractor = resolve_object_name(
290 feature_extractor_name,
291 gv.file_storage_data_mapping[gv.extractor_nickname_list_path],
292 gv.settings().DEFAULT_extractor_dir, class_name=Extractor)
293 feature_extractor_name = feature_extractor_name.replace("_", " ") # Latex
294 feature_extractor_latex_list.add_item(
295 pl.UnsafeCommand(f"textbf{{{feature_extractor_name}}} "
296 f"({extractor.output_dimension} features)"))
297 # Report Training results
298 report.append(pl.Subsection("Training Results"))
299 # 1. Report VBS and selector performance, create ranking list of the solvers
300 # TODO Add ref here to the training sets section?
301 report.append(f"In this section, the {scenario.objective.name} results for the "
302 "portfolio selector on solving the training instance set(s) listed "
303 "is reported. ")
304 report.append(f"The {scenario.objective.name} values for the Virtual Best Solver "
305 "(VBS), i.e., the perfect portfolio selector is ")
306 report.append(pl.utils.bold(f"{round(scenario_output.vbs_performance, MAX_DEC)}"))
307 report.append(", the actual portfolio selector performance is ")
308 report.append(
309 pl.utils.bold(f"{round(scenario_output.actual_performance, MAX_DEC)}.\n"))
311 report.append("Below, the solvers are ranked based on "
312 f"{scenario.objective.name} performance:")
313 with report.create(pl.Enumerate()) as ranking_list:
314 for solver_name, conf_id, value in scenario_output.solver_performance_ranking:
315 value = round(value, MAX_DEC)
316 solver_name = solver_name.replace("_", " ") # Latex fix
317 conf_id = conf_id.replace("_", " ") # Latex fix
318 ranking_list.add_item(
319 pl.UnsafeCommand(f"textbf{{{solver_name}}} ({conf_id}): {value}"))
321 # 2. Marginal contribution ranking list VBS
322 report.append(pl.Subsubsection("Marginal Contribution Ranking List"))
323 report.append(
324 "The following list shows the marginal contribution ranking list for the VBS:")
325 with report.create(pl.Enumerate()) as ranking_list:
326 for (solver_name, conf_id,
327 contribution, performance) in scenario_output.marginal_contribution_perfect:
328 contribution, performance =\
329 round(contribution, MAX_DEC), round(performance, MAX_DEC)
330 solver_name = solver_name.replace("_", " ") # Latex fix
331 conf_id = conf_id.replace("_", " ") # Latex fix
332 ranking_list.add_item(pl.UnsafeCommand(
333 f"textbf{{{solver_name}}} ({conf_id}): {contribution} ({performance})"))
335 # 3. Marginal contribution ranking list actual selector
336 report.append("The following list shows the marginal contribution ranking list for "
337 "the actual portfolio selector:")
338 with report.create(pl.Enumerate()) as ranking_list:
339 for (solver_name, conf_id,
340 contribution, performance) in scenario_output.marginal_contribution_actual:
341 contribution, performance =\
342 round(contribution, MAX_DEC), round(performance, MAX_DEC)
343 solver_name = solver_name.replace("_", " ") # Latex fix
344 conf_id = conf_id.replace("_", " ") # Latex fix
345 ranking_list.add_item(pl.UnsafeCommand(
346 f"textbf{{{solver_name}}} ({conf_id}): {contribution} ({performance})"))
348 # 4. Create scatter plot analysis
349 report.append(pl.Subsubsection("Scatter Plot Analysis"))
350 report.append(latex.AutoRef(f"fig:sbsvsselector{time_stamp}"))
351 report.append(pl.utils.bold(" ")) # Trick to force a white space
352 report.append("shows the empirical comparison between the portfolio "
353 "selector and the single best solver (SBS). ")
354 report.append(latex.AutoRef("fig:vbsvsselector"))
355 report.append(pl.utils.bold(" ")) # Trick to force a white space
356 report.append("shows the empirical comparison between the actual portfolio selector "
357 "and the virtual best solver (VBS).")
358 # Create figure on SBS versus the selector
359 sbs_name, sbs_config, _ = scenario_output.solver_performance_ranking[0]
360 # sbs_plot_name = f"{Path(sbs_name).name} ({sbs_config})"
361 sbs_performance = scenario_output.sbs_performance
362 selector_performance = scenario_output.actual_performance_data
364 # Join the data together
365 import pandas as pd
366 df = pd.DataFrame(
367 [sbs_performance, selector_performance],
368 index=[f"{Path(sbs_name).name} ({sbs_config})", "Selector"], dtype=float).T
369 plot = latex.comparison_plot(df, "Single Best Solver vs Selector")
370 plot_path = plot_dir /\
371 f"{Path(sbs_name).name}_{sbs_config}_vs_"\
372 f"Selector_{scenario.selector.model_class.__name__}.pdf"
373 plot.write_image(plot_path, width=500, height=500)
374 with report.create(pl.Figure()) as figure:
375 figure.add_image(str(plot_path.relative_to(report_dir)),
376 width=pl.utils.NoEscape(r"0.6\textwidth"))
377 figure.add_caption("Empirical comparison between the Single Best Solver and the "
378 "Selector")
379 label = r"label{fig:sbsvsselector" + str(time_stamp) + r"}"
380 figure.append(pl.UnsafeCommand(f"{label}"))
382 # Comparison between the actual portfolio selector in Sparkle and the VBS.
383 vbs_performance = scenario_output.vbs_performance_data.tolist()
384 df = pd.DataFrame([vbs_performance, selector_performance],
385 index=["Virtual Best Solver", "Selector"], dtype=float).T
386 plot = latex.comparison_plot(df, "Virtual Best Solver vs Selector")
387 plot_path = plot_dir /\
388 f"Virtual_Best_Solver_vs_Selector_{scenario.selector.model_class.__name__}.pdf"
389 plot.write_image(plot_path, width=500, height=500)
390 with report.create(pl.Figure()) as figure:
391 figure.add_image(str(plot_path.relative_to(report_dir)),
392 width=pl.utils.NoEscape(r"0.6\textwidth"))
393 figure.add_caption(
394 "Empirical comparison between the Virtual Best Solver and the Selector")
395 figure.append(pl.UnsafeCommand(r"label{fig:vbsvsselector}"))
397 if scenario_output.test_sets:
398 report.append(pl.Subsection("Test Results"))
399 report.append("The following results are reported on the test set(s):")
400 with report.create(pl.Itemize()) as latex_list:
401 for test_set_name, test_set_size in scenario_output.test_sets:
402 result = round(scenario_output.test_set_performance[test_set_name],
403 MAX_DEC)
404 latex_list.add_item(pl.UnsafeCommand(
405 f"textbf{{{test_set_name}}} ({test_set_size} instances): {result}"))
408def generate_parallel_portfolio_section(report: pl.Document,
409 scenario: PerformanceDataFrame) -> None:
410 """Generate a section for a parallel portfolio scenario."""
411 report_dir = Path(report.default_filepath).parent
412 portfolio_name = scenario.csv_filepath.parent.name
413 time_stamp = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))
414 plot_dir = report_dir / f"{portfolio_name.replace(' ', '_')}_plots_{time_stamp}"
415 plot_dir.mkdir()
416 report.append(pl.Section(f"Parallel Portfolio {portfolio_name}"))
417 report.append(
418 "In this scenario, Sparkle runs the portfolio of Solvers on each instance in "
419 "parallel with "
420 f"{gv.settings().get_parallel_portfolio_number_of_seeds_per_solver()} different "
421 "seeds. The cutoff time for each solver run is set to "
422 f"{gv.settings().get_general_solver_cutoff_time()} seconds.")
423 report.append(pl.Subsection("Solvers & Instance Sets"))
424 report.append("The following Solvers were used in the portfolio:")
425 # 1. Report on the Solvers and Instance Sets used for the portfolio
426 with report.create(pl.Itemize()) as solver_latex_list:
427 configs = scenario.configurations
428 for solver in scenario.solvers:
429 solver_name = solver.replace("_", " ")
430 solver_latex_list.add_item(
431 pl.UnsafeCommand(
432 f"textbf{{{solver_name}}} "
433 f"({len(configs[solver])} configurations)"))
434 report.append("The following Instance Sets were used in the portfolio:")
435 instance_sets = set(Path(instance).parent.name for instance in scenario.instances)
436 instance_set_count = [
437 len([i for i in scenario.instances if Path(i).parent.name == s])
438 for s in instance_sets]
439 with report.create(pl.Itemize()) as instance_set_latex_list:
440 for set_name, set_size in zip(instance_sets, instance_set_count):
441 set_name = set_name.replace("_", " ") # Latex fix
442 instance_set_latex_list.add_item(
443 pl.UnsafeCommand(
444 f"textbf{{{set_name}}} ({set_size} instances)"))
445 # 2. List which solver was the best on how many instances
446 report.append(pl.Subsection("Portfolio Performance"))
447 objective = scenario.objectives[0]
448 report.append(f"The objective for the portfolio is {objective}. The "
449 "following performance of the solvers was found over the instances: ")
450 best_solver_count = {solver: 0 for solver in scenario.solvers}
451 for instance in scenario.instances:
452 ranking = scenario.get_solver_ranking(objective=objective, instances=[instance])
453 best_solver_count[ranking[0][0]] += 1
455 with report.create(pl.Itemize()) as latex_list:
456 for solver, count in best_solver_count.items():
457 solver_name = solver.replace("_", " ")
458 latex_list.add_item(pl.UnsafeCommand(
459 f"textbf{{{solver_name}}} was the best solver on {count} instance(s)."))
460 # TODO Report how many instances remained unsolved
462 # 3. Create table showing the performance of the portfolio vs and all solvers,
463 # by showing the status count and number of times the solver was best
464 solver_cancelled_count = {solver: 0 for solver in scenario.solvers}
465 solver_timeout_count = {solver: 0 for solver in scenario.solvers}
466 status_objective = [o for o in scenario.objective_names
467 if o.lower().startswith("status")][0]
468 cancelled_status = [SolverStatus.UNKNOWN, SolverStatus.CRASHED, SolverStatus.WRONG,
469 SolverStatus.ERROR, SolverStatus.KILLED]
470 for solver in scenario.solvers:
471 status = scenario.get_value(solver=solver, objective=status_objective)
472 for status in scenario.get_value(solver=solver, objective=status_objective):
473 status = SolverStatus(status)
474 if status in cancelled_status:
475 solver_cancelled_count[solver] += 1
476 elif status == SolverStatus.TIMEOUT:
477 solver_timeout_count[solver] += 1
479 report.append(latex.AutoRef("tab:parallelportfoliotable"))
480 report.append(pl.utils.bold(" "))
481 report.append(" shows the performance of the portfolio on the test set(s).")
482 tabular = pl.Tabular("r|rrrr")
483 tabular.add_row(["Solver", objective, "# Timeouts", "# Cancelled", "# Best"])
484 tabular.add_hline()
485 solver_performance = {solver: round(performance, MAX_DEC)
486 for solver, _, performance in
487 scenario.get_solver_ranking(objective=objective)}
488 for solver in scenario.solvers:
489 tabular.add_row(solver,
490 solver_performance[solver],
491 solver_timeout_count[solver],
492 solver_cancelled_count[solver],
493 best_solver_count[solver])
494 tabular.add_hline()
495 portfolio_performance = round(
496 scenario.best_performance(objective=objective), MAX_DEC)
497 tabular.add_row(portfolio_name, portfolio_performance,
498 sum(solver_timeout_count.values()),
499 sum(solver_cancelled_count.values()),
500 sum(best_solver_count.values()))
501 table_portfolio = pl.Table(position="h")
502 table_portfolio.append(pl.UnsafeCommand("centering"))
503 table_portfolio.append(tabular)
504 table_portfolio.add_caption("Parallel Portfolio Performance")
505 table_portfolio.append(pl.UnsafeCommand(r"label{tab:parallelportfoliotable}"))
506 report.append(table_portfolio)
508 # 4. Create scatter plot analysis between the portfolio and the single best solver
509 sbs_name = scenario.get_solver_ranking(objective=objective)[0][0]
510 sbs_instance_performance = scenario.get_value(
511 solver=sbs_name, objective=objective.name)
512 sbs_name = Path(sbs_name).name
513 report.append(latex.AutoRef("fig:portfoliovssbs"))
514 report.append(pl.utils.bold(" "))
515 report.append(" shows the emprical comparison between the portfolio and the single "
516 f"best solver (SBS) {sbs_name}.")
517 portfolio_instance_performance = scenario.best_instance_performance(
518 objective=objective.name).tolist()
519 import pandas as pd
520 df = pd.DataFrame(
521 [sbs_instance_performance, portfolio_instance_performance],
522 index=[f"SBS ({sbs_name}) Performance", "Portfolio Performance"], dtype=float).T
523 plot = latex.comparison_plot(df, None)
524 plot_path = plot_dir /\
525 f"sbs_{sbs_name}_vs_"\
526 f"parallel_portfolio.pdf"
527 plot.write_image(plot_path, width=500, height=500)
528 with report.create(pl.Figure(position="h")) as figure:
529 figure.add_image(str(plot_path.relative_to(report_dir)),
530 width=pl.utils.NoEscape(r"0.6\textwidth"))
531 figure.add_caption(
532 f"Portfolio vs SBS Performance ({objective})")
533 figure.append(pl.UnsafeCommand(
534 r"label{fig:portfoliovssbs}"))
537def generate_appendix(report: pl.Document,
538 performance_data: PerformanceDataFrame,
539 feature_data: FeatureDataFrame) -> None:
540 """Generate an appendix for the report."""
541 # report.append(pl.UnsafeCommand("appendix"))
542 # report.append("Below are the full performance and feature data frames.")
543 # TODO: Add long table for the entire performance data frame
544 # performance_data.to_latex() # maybe?
545 # TODO: Add long table for the entire feature data frame
548def main(argv: list[str]) -> None:
549 """Generate a report for executed experiments in the platform."""
550 # Log command call
551 sl.log_command(sys.argv)
553 # Define command line arguments
554 parser = parser_function()
556 # Process command line arguments
557 args = parser.parse_args(argv)
559 performance_data = PerformanceDataFrame(gv.settings().DEFAULT_performance_data_path)
560 feature_data = FeatureDataFrame(gv.settings().DEFAULT_feature_data_path)
562 # Fetch all known scenarios
563 configuration_scenarios = gv.configuration_scenarios(refresh=True)
564 selection_scenarios = gv.selection_scenarios(refresh=True)
565 parallel_portfolio_scenarios = gv.parallel_portfolio_scenarios()
567 # Filter scenarios based on args
568 if args.solvers:
569 solvers = [resolve_object_name(s,
570 gv.solver_nickname_mapping,
571 gv.settings().DEFAULT_solver_dir,
572 Solver)
573 for s in args.solvers]
574 configuration_scenarios = [
575 s for s in configuration_scenarios
576 if s.solver.directory in [s.directory for s in solvers]
577 ]
578 selection_scenarios = [
579 s for s in selection_scenarios
580 if set(s.solvers).intersection([str(s.directory) for s in solvers])
581 ]
582 parallel_portfolio_scenarios = [
583 s for s in parallel_portfolio_scenarios
584 if set(s.solvers).intersection([str(s.directory) for s in solvers])
585 ]
586 if args.instance_sets:
587 instance_sets = [resolve_object_name(s,
588 gv.instance_set_nickname_mapping,
589 gv.settings().DEFAULT_instance_dir,
590 Instance_Set)
591 for s in args.instance_sets]
592 configuration_scenarios = [
593 s for s in configuration_scenarios
594 if s.instance_set.directory in [s.directory for s in instance_sets]
595 ]
596 selection_scenarios = [
597 s for s in selection_scenarios
598 if set(s.instance_sets).intersection([str(s.name) for s in instance_sets])
599 ]
600 parallel_portfolio_scenarios = [
601 s for s in parallel_portfolio_scenarios
602 if set(s.instance_sets).intersection([str(s.name) for s in instance_sets])
603 ]
605 processed_configuration_scenarios = []
606 processed_selection_scenarios = []
607 possible_test_sets = [Instance_Set(p)
608 for p in gv.settings().DEFAULT_instance_dir.iterdir()]
609 for configuration_scenario in configuration_scenarios:
610 processed_configuration_scenarios.append(
611 (ConfigurationOutput(
612 configuration_scenario, performance_data, possible_test_sets),
613 configuration_scenario))
614 for selection_scenario in selection_scenarios:
615 processed_selection_scenarios.append(
616 (SelectionOutput(selection_scenario,
617 feature_data), selection_scenario))
619 raw_output = gv.settings().DEFAULT_output_analysis / "JSON"
620 if raw_output.exists(): # Clean
621 shutil.rmtree(raw_output)
622 raw_output.mkdir()
624 # Write JSON
625 output_json = {}
626 for output, configuration_scenario in processed_configuration_scenarios:
627 output_json[configuration_scenario.name] = output.serialise()
628 for output, selection_scenario in processed_selection_scenarios:
629 output_json[selection_scenario.name] = output.serialise()
630 # TODO: We do not have an output object for parallel portfolios
632 raw_output_json = raw_output / "output.json"
633 with raw_output_json.open("w") as f:
634 json.dump(output_json, f, indent=4)
636 print(f"Machine readable output written to: {raw_output_json}")
638 if args.only_json: # Done
639 sys.exit(0)
641 # TODO: Group scenarios based on:
642 # - Configuration / Selection / Parallel Portfolio
643 # - Training Instance Set / Testing Instance Set
644 # - Configurators can be merged as long as we can match their budgets clearly
645 report_directory = gv.settings().DEFAULT_output_analysis / "report"
646 if report_directory.exists(): # Clean it
647 shutil.rmtree(report_directory)
648 report_directory.mkdir()
649 target_path = report_directory / "report"
650 report = pl.document.Document(
651 default_filepath=str(target_path),
652 document_options=["british"])
653 bibpath = gv.settings().bibliography_path
654 newbibpath = report_directory / "report.bib"
655 shutil.copy(bibpath, newbibpath)
656 # BUGFIX for unknown package load in PyLatex
657 p = pl.package.Package("lastpage")
658 if p in report.packages:
659 report.packages.remove(p)
660 report.packages.append(pl.package.Package("geometry",
661 options=["verbose",
662 "tmargin=3.5cm",
663 "bmargin=3.5cm",
664 "lmargin=3cm",
665 "rmargin=3cm"]))
666 # Unsafe command for \emph{Sparkle}
667 report.preamble.extend([
668 pl.UnsafeCommand(
669 "title",
670 r"\emph{Sparkle} Algorithm Portfolio report"),
671 pl.UnsafeCommand(
672 "author",
673 r"Generated by \emph{Sparkle} "
674 f"(version: {__sparkle_version__})")])
675 report.append(pl.Command("maketitle"))
676 report.append(pl.Section("Introduction"))
677 # TODO: A quick overview to the introduction on whats considered in the report
678 # regarding Solvers, Instance Sets and Feature Extractors
679 report.append(pl.UnsafeCommand(
680 r"emph{Sparkle}~\cite{Hoos15} is a multi-agent problem-solving platform based on"
681 r" Programming by Optimisation (PbO)~\cite{Hoos12}, and would provide a number "
682 "of effective algorithm optimisation techniques (such as automated algorithm "
683 "configuration, portfolio-based algorithm selection, etc.) to accelerate the "
684 "existing solvers."))
686 for (scenario_output, scenario) in processed_configuration_scenarios:
687 generate_configuration_section(report, scenario, scenario_output)
689 for (scenario_output, scenario) in processed_selection_scenarios:
690 generate_selection_section(report, scenario, scenario_output)
692 for parallel_dataframe in parallel_portfolio_scenarios:
693 generate_parallel_portfolio_section(report, parallel_dataframe)
695 generate_appendix(report, performance_data, feature_data)
697 # Adding bibliography
698 report.append(pl.NewPage()) # Ensure it starts on new page
699 report.append(pl.Command("bibliographystyle", arguments=["plain"]))
700 report.append(pl.Command("bibliography", arguments=[str(newbibpath)]))
701 # Generate the report .tex and .pdf
702 report.generate_pdf(target_path, clean=False, clean_tex=False, compiler="pdflatex")
703 # TODO: This should be done by PyLatex. Generate the bib and regenerate the report
704 # Reference for the (terrible) solution: https://tex.stackexchange.com/
705 # questions/63852/question-mark-or-bold-citation-key-instead-of-citation-number
706 import subprocess
707 # Run BibTex silently
708 subprocess.run(["bibtex", newbibpath.with_suffix("")],
709 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
710 report.generate_pdf(target_path, clean=False, clean_tex=False, compiler="pdflatex")
711 report.generate_pdf(target_path, clean=False, clean_tex=False, compiler="pdflatex")
712 print(f"Report generated at {target_path}.pdf")
713 sys.exit(0)
716if __name__ == "__main__":
717 main(sys.argv[1:])