Coverage for src/sparkle/CLI/generate_report.py: 66%

1#!/usr/bin/env python3 

2"""Sparkle command to generate a report for an executed experiment.""" 

3 

4import sys 

5import shutil 

6import argparse 

7from pathlib import Path 

8import time 

9import json 

10import pandas as pd 

11 

12from pylatex import NoEscape, NewPage 

13import pylatex as pl 

14from sparkle import __version__ as __sparkle_version__ 

15 

16from sparkle.CLI.help import global_variables as gv 

17from sparkle.CLI.help import resolve_object_name 

18from sparkle.CLI.help import logging as sl 

19from sparkle.CLI.help import argparse_custom as ac 

20 

21from sparkle.solver import Solver 

22from sparkle.instance import Instance_Set 

23from sparkle.selector import Extractor 

24from sparkle.structures import PerformanceDataFrame, FeatureDataFrame 

25from sparkle.configurator.configurator import ConfigurationScenario 

26from sparkle.selector.selector import SelectionScenario 

27from sparkle.types import SolverStatus 

28from sparkle.platform import Settings 

29 

30from sparkle.platform import latex 

31from sparkle.platform.output.configuration_output import ConfigurationOutput 

32from sparkle.platform.output.selection_output import SelectionOutput 

33 

34 

35MAX_DEC = 4 # Maximum decimals used for each reported value 

36MAX_COLS_PER_TABLE = 2 # number of value columns extra to number of key columns 

37WIDE_TABLE_THRESHOLD = 4 # columns above which we switch to landscape 

38NUM_KEYS_PDF = 3 

39NUM_KEYS_FDF = 3 

40MAX_CELL_LEN = 17 

41 

42 

43def parser_function() -> argparse.ArgumentParser: 

44 """Define the command line arguments.""" 

45 parser = argparse.ArgumentParser( 

46 description="Generates a report for all known selection, configuration and " 

47 "parallel portfolio scenarios will be generated.", 

48 epilog="If you wish to filter specific solvers, instance sets, ... have a look " 

49 "at the command line arguments.", 

50 ) 

51 # Add argument for filtering solvers 

52 parser.add_argument( 

53 *ac.SolversReportArgument.names, **ac.SolversReportArgument.kwargs 

54 ) 

55 # Add argument for filtering instance sets 

56 parser.add_argument( 

57 *ac.InstanceSetsReportArgument.names, **ac.InstanceSetsReportArgument.kwargs 

58 ) 

59 

60 # Add argument for filtering appendix 

61 parser.add_argument( 

62 *Settings.OPTION_appendices.args, **Settings.OPTION_appendices.kwargs 

63 ) 

64 

65 # Add argument for filtering configurators? 

66 # Add argument for filtering selectors? 

67 # Add argument for filtering ??? scenario ids? configuration ids? 

68 parser.add_argument(*ac.GenerateJSONArgument.names, **ac.GenerateJSONArgument.kwargs) 

69 return parser 

70 

71 

72def generate_configuration_section( 

73 report: pl.Document, 

74 scenario: ConfigurationScenario, 

75 scenario_output: ConfigurationOutput, 

76) -> None: 

77 """Generate a section for a configuration scenario.""" 

78 report_dir = Path(report.default_filepath).parent 

79 time_stamp = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())) 

80 plot_dir = ( 

81 report_dir 

82 / f"{scenario.configurator.__name__}_{scenario.name}_plots_{time_stamp}" 

83 ) 

84 plot_dir.mkdir(exist_ok=True) 

85 

86 # 1. Write section intro 

87 report.append( 

88 pl.Section( 

89 f"{scenario.configurator.__name__} Configuration: " 

90 f"{scenario.solver.name} on {scenario.instance_set.name}" 

91 ) 

92 ) 

93 report.append("In this scenario, ") 

94 report.append( 

95 pl.UnsafeCommand( 

96 f"textbf{{{scenario.configurator.__name__}}} " 

97 f"({scenario.configurator.full_name})~\\cite" 

98 f"{{{scenario.configurator.__name__}}} with version " 

99 f"{scenario.configurator.version} was used for configuration. " 

100 ) 

101 ) 

102 report.append( 

103 f"The Solver {scenario.solver} was optimised on training set " 

104 f"{scenario.instance_set}. The scenario was run {scenario.number_of_runs} " 

105 f"times independently with different seeds, yielding {scenario.number_of_runs} " 

106 f"configurations. The cutoff time for the solver was set to " 

107 f"{scenario.solver_cutoff_time} seconds. The optimised objective is " 

108 f"{scenario.sparkle_objectives[0]}. Each Configuration was evaluated on the " 

109 "training set to determine the best configuration, e.g. the best " 

110 f"{scenario.sparkle_objectives[0]} value on the training set." 

111 ) 

112 

113 # 2. Report all the configurator settings in table format 

114 report.append(pl.Subsection("Configurator Settings")) 

115 report.append( 

116 f"The following settings were used for {scenario.configurator.__name__}:\n" 

117 ) 

118 tabular = pl.Tabular("l|r") 

119 tabular.add_row("Setting", "Value") 

120 tabular.add_hline() 

121 for setting, value in scenario.serialise().items(): 

122 # Keep only the last path segment for paths 

123 # Otherwise tables get too wide and we can't see other values 

124 stripped_value = str(value).strip().replace("\\", "/") 

125 segments = [segment for segment in stripped_value.split("/") if segment] 

126 if segments[-1]: 

127 tabular.add_row([setting, segments[-1]]) 

128 else: 

129 tabular.add_row([setting, "None"]) 

130 table_conf_settings = pl.Table(position="h") 

131 table_conf_settings.append(pl.UnsafeCommand("centering")) 

132 table_conf_settings.append(tabular) 

133 table_conf_settings.add_caption("Configurator Settings") 

134 report.append(table_conf_settings) 

135 

136 # 3. Report details on instance and solver used 

137 report.append(pl.Subsection("Solver & Instance Set(s) Details")) 

138 cs = scenario_output.solver.get_configuration_space() 

139 report.append( 

140 f"The solver {scenario_output.solver} was configured using " 

141 f"{len(cs.values())} configurable (hyper)parameters. " 

142 f"The configuration space has {len(cs.conditions)} conditions. " 

143 ) 

144 report.append("The following instance sets were used for the scenario:") 

145 with report.create(pl.Itemize()) as instance_set_latex_list: 

146 for instance_set in [ 

147 scenario_output.instance_set_train 

148 ] + scenario_output.test_instance_sets: 

149 training_set_name = instance_set.name.replace("_", " ") # Latex fix 

150 instance_set_latex_list.add_item( 

151 pl.UnsafeCommand( 

152 f"textbf{{{training_set_name}}} ({instance_set.size} instances)" 

153 ) 

154 ) 

155 

156 # Function to generate a results summary of default vs best on an instance set 

157 def instance_set_summary(instance_set_name: str) -> None: 

158 """Generate a results summary of default vs best on an instance set.""" 

159 instance_set_results = scenario_output.instance_set_results[instance_set_name] 

160 report.append( 

161 f"The {scenario.sparkle_objectives[0]} value of the Default " 

162 f"Configuration on {instance_set_name} was " 

163 ) 

164 report.append( 

165 pl.UnsafeCommand( 

166 f"textbf{{{round(instance_set_results.default_performance, MAX_DEC)}}}.\n" 

167 ) 

168 ) 

169 report.append( 

170 f"The {scenario.sparkle_objectives[0]} value of the Best " 

171 f"Configuration on {instance_set_name} was " 

172 ) 

173 report.append( 

174 pl.UnsafeCommand( 

175 f"textbf{{{round(instance_set_results.best_performance, MAX_DEC)}}}.\n" 

176 ) 

177 ) 

178 report.append("In ") 

179 report.append(latex.AutoRef(f"fig:bestvsdefault{instance_set_name}{time_stamp}")) 

180 report.append(pl.utils.bold(" ")) # Force white space 

181 report.append("the results are plotted per instance.") 

182 # Create graph to compare best configuration vs default on the instance set 

183 

184 df = pd.DataFrame( 

185 [ 

186 instance_set_results.default_instance_performance, 

187 instance_set_results.best_instance_performance, 

188 ], 

189 index=["Default Configuration", "Best Configuration"], 

190 dtype=float, 

191 ).T 

192 plot = latex.comparison_plot(df, None) 

193 plot_path = ( 

194 plot_dir / f"{scenario_output.best_configuration_key}_vs_" 

195 f"Default_{instance_set_name}.pdf" 

196 ) 

197 plot.write_image(plot_path, width=500, height=500) 

198 with report.create(pl.Figure(position="h")) as figure: 

199 figure.add_image( 

200 str(plot_path.relative_to(report_dir)), 

201 width=pl.utils.NoEscape(r"0.6\textwidth"), 

202 ) 

203 figure.add_caption( 

204 f"Best vs Default Performance on {instance_set_name} " 

205 f"({scenario.sparkle_objectives[0]})" 

206 ) 

207 figure.append( 

208 pl.UnsafeCommand( 

209 r"label{" 

210 f"fig:bestvsdefault{instance_set_name}{time_stamp}" 

211 r"}" 

212 ) 

213 ) 

214 if scenario.sparkle_objectives[0].time: # Write status table 

215 report.append("The following Solver status were found per instance:") 

216 tabular = pl.Tabular("l|c|c|c") 

217 tabular.add_row("Status", "Default", "Best", "Overlap") 

218 tabular.add_hline() 

219 # Count the statuses 

220 for status in SolverStatus: 

221 default_count, best_count, overlap_count = 0, 0, 0 

222 for instance in instance_set_results.instance_status_default.keys(): 

223 instance = str(instance) 

224 default_hit = ( 

225 instance_set_results.instance_status_default[instance] == status 

226 ) 

227 best_hit = ( 

228 instance_set_results.instance_status_best[instance] == status 

229 ) 

230 default_count += default_hit 

231 best_count += best_hit 

232 overlap_count += default_hit and best_hit 

233 if default_count or best_count: 

234 tabular.add_row(status, default_count, best_count, overlap_count) 

235 table_status_values = pl.Table(position="h") 

236 table_status_values.append(pl.UnsafeCommand("centering")) 

237 table_status_values.append(tabular) 

238 table_status_values.add_caption( 

239 "Status count for the best and default configuration." 

240 ) 

241 report.append(table_status_values) 

242 

243 # 4. Report the results of the best configuration on the training set vs the default 

244 report.append( 

245 pl.Subsection( 

246 f"Comparison of Default and Best Configuration on Training Set " 

247 f"{scenario_output.instance_set_train.name}" 

248 ) 

249 ) 

250 instance_set_summary(scenario_output.instance_set_train.name) 

251 

252 # 5. Report the actual config values 

253 report.append(pl.Subsubsection("Best Configuration Values")) 

254 if ( 

255 scenario_output.best_configuration_key 

256 == PerformanceDataFrame.default_configuration 

257 ): 

258 report.append( 

259 "The configurator failed to find a better configuration than the " 

260 "default configuration on the training set in this scenario." 

261 ) 

262 else: 

263 report.append( 

264 "The following parameter values " 

265 "were found to be the best on the training set:\n" 

266 ) 

267 tabular = pl.Tabular("l|r") 

268 tabular.add_row("Parameter", "Value") 

269 tabular.add_hline() 

270 for parameter, value in scenario_output.best_configuration.items(): 

271 tabular.add_row([parameter, str(value)]) 

272 table_best_values = pl.Table(position="h") 

273 table_best_values.append(pl.UnsafeCommand("centering")) 

274 table_best_values.append(tabular) 

275 table_best_values.add_caption("Best found configuration values") 

276 report.append(table_best_values) 

277 

278 # 6. Report the results of best vs default conf on the test sets 

279 

280 for test_set in scenario_output.test_instance_sets: 

281 report.append( 

282 pl.Subsection( 

283 f"Comparison of Default and Best Configuration on Test Set " 

284 f"{test_set.name}" 

285 ) 

286 ) 

287 instance_set_summary(test_set.name) 

288 

289 # 7. Report the parameter ablation scenario if present 

290 if scenario.ablation_scenario: 

291 report.append(pl.Subsection("Parameter importance via Ablation")) 

292 report.append("Ablation analysis ") 

293 report.append(pl.UnsafeCommand(r"cite{FawcettHoos16} ")) 

294 test_set = scenario.ablation_scenario.test_set 

295 if not scenario.ablation_scenario.test_set: 

296 test_set = scenario.ablation_scenario.train_set 

297 report.append( 

298 f"is performed from the default configuration of {scenario.solver} to the " 

299 f"best found configuration ({scenario_output.best_configuration_key}) " 

300 "to see which parameter changes between them contribute most to the improved" 

301 " performance. The ablation path uses the training set " 

302 f"{scenario.ablation_scenario.train_set.name} and validation is performed " 

303 f"on the test set {test_set.name}. The set of parameters that differ in the " 

304 "two configurations will form the ablation path. Starting from the default " 

305 "configuration, the path is computed by performing a sequence of rounds. In" 

306 " a round, each available parameter is flipped in the configuration and is " 

307 "validated on its performance. The flipped parameter with the best " 

308 "performance in that round, is added to the configuration and the next round" 

309 " starts with the remaining parameters. This repeats until all parameters " 

310 "are flipped, which is the best found configuration. The analysis resulted " 

311 "in the ablation presented in " 

312 ) 

313 report.append(latex.AutoRef("tab:ablationtable")) 

314 report.append(".") 

315 

316 # Add ablation table 

317 tabular = pl.Tabular("r|l|r|r|r") 

318 data = scenario.ablation_scenario.read_ablation_table() 

319 for index, row in enumerate(data): 

320 tabular.add_row(*row) 

321 if index == 0: 

322 tabular.add_hline() 

323 table_ablation = pl.Table(position="h") 

324 table_ablation.append(pl.UnsafeCommand("centering")) 

325 table_ablation.append(tabular) 

326 table_ablation.add_caption("Ablation table") 

327 table_ablation.append(pl.UnsafeCommand(r"label{tab:ablationtable}")) 

328 report.append(table_ablation) 

329 

330 

331def generate_selection_section( 

332 report: pl.Document, scenario: SelectionScenario, scenario_output: SelectionOutput 

333) -> None: 

334 """Generate a section for a selection scenario.""" 

335 report_dir = Path(report.default_filepath).parent 

336 time_stamp = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())) 

337 plot_dir = report_dir / f"{scenario.name.replace(' ', '_')}_plots_{time_stamp}" 

338 plot_dir.mkdir(exist_ok=True) 

339 report.append( 

340 pl.Section( 

341 f"Selection: {scenario.selector.model_class.__name__} on " 

342 f"{' '.join([s[0] for s in scenario_output.training_instance_sets])}" 

343 ) 

344 ) 

345 report.append( 

346 f"In this scenario, a {scenario.selector.model_class.__name__} " 

347 f" ({scenario.selector.selector_class.__name__}) was trained on the " 

348 "performance and feature data using ASF-lib. The following solvers " 

349 f"were run with a cutoff time of {scenario.solver_cutoff} seconds:" 

350 ) 

351 with report.create(pl.Itemize()) as solver_latex_list: 

352 for solver_name in scenario_output.solvers.keys(): 

353 solver_name = solver_name.replace("_", " ") 

354 solver_latex_list.add_item( 

355 pl.UnsafeCommand( 

356 f"textbf{{{solver_name}}} " 

357 f"({len(scenario_output.solvers[solver_name])} configurations)" 

358 ) 

359 ) 

360 # Report training instance sets 

361 report.append("The following training instance sets were used:") 

362 with report.create(pl.Itemize()) as instance_set_latex_list: 

363 for training_set_name, set_size in scenario_output.training_instance_sets: 

364 training_set_name = training_set_name.replace("_", " ") # Latex fix 

365 instance_set_latex_list.add_item( 

366 pl.UnsafeCommand(f"textbf{{{training_set_name}}} ({set_size} instances)") 

367 ) 

368 # Report feature extractors 

369 report.append( 

370 "The following feature extractors were used with a extractor cutoff " 

371 f"time of {scenario.extractor_cutoff} seconds:" 

372 ) 

373 with report.create(pl.Itemize()) as feature_extractor_latex_list: 

374 for feature_extractor_name in scenario.feature_extractors: 

375 extractor = resolve_object_name( 

376 feature_extractor_name, 

377 gv.file_storage_data_mapping[gv.extractor_nickname_list_path], 

378 gv.settings().DEFAULT_extractor_dir, 

379 class_name=Extractor, 

380 ) 

381 if extractor is not None: 

382 output_dimension = extractor.output_dimension 

383 else: 

384 extractor_name = Path(feature_extractor_name).name 

385 output_dimension = sum( 

386 column.startswith(f"{extractor_name}_") 

387 or column.endswith(f"_{extractor_name}") 

388 for column in scenario.feature_data.columns 

389 ) 

390 feature_extractor_name = feature_extractor_name.replace("_", " ") # Latex 

391 feature_extractor_latex_list.add_item( 

392 pl.UnsafeCommand( 

393 f"textbf{{{feature_extractor_name}}} ({output_dimension} features)" 

394 ) 

395 ) 

396 # Report Training results 

397 report.append(pl.Subsection("Training Results")) 

398 # 1. Report VBS and selector performance, create ranking list of the solvers 

399 # TODO Add ref here to the training sets section? 

400 report.append( 

401 f"In this section, the {scenario.objective.name} results for the " 

402 "portfolio selector on solving the training instance set(s) listed " 

403 "is reported. " 

404 ) 

405 report.append( 

406 f"The {scenario.objective.name} values for the Virtual Best Solver " 

407 "(VBS), i.e., the perfect portfolio selector is " 

408 ) 

409 report.append(pl.utils.bold(f"{round(scenario_output.vbs_performance, MAX_DEC)}")) 

410 report.append(", the actual portfolio selector performance is ") 

411 report.append( 

412 pl.utils.bold(f"{round(scenario_output.actual_performance, MAX_DEC)}.\n") 

413 ) 

414 

415 report.append( 

416 f"Below, the solvers are ranked based on {scenario.objective.name} performance:" 

417 ) 

418 with report.create(pl.Enumerate()) as ranking_list: 

419 for solver_name, conf_id, value in scenario_output.solver_performance_ranking: 

420 value = round(value, MAX_DEC) 

421 solver_name = solver_name.replace("_", " ") # Latex fix 

422 conf_id = conf_id.replace("_", " ") # Latex fix 

423 ranking_list.add_item( 

424 pl.UnsafeCommand(f"textbf{{{solver_name}}} ({conf_id}): {value}") 

425 ) 

426 

427 # 2. Marginal contribution ranking list VBS 

428 report.append(pl.Subsubsection("Marginal Contribution Ranking List")) 

429 report.append( 

430 "The following list shows the marginal contribution ranking list for the VBS:" 

431 ) 

432 with report.create(pl.Enumerate()) as ranking_list: 

433 for ( 

434 solver_name, 

435 conf_id, 

436 contribution, 

437 performance, 

438 ) in scenario_output.marginal_contribution_perfect: 

439 contribution, performance = ( 

440 round(contribution, MAX_DEC), 

441 round(performance, MAX_DEC), 

442 ) 

443 solver_name = solver_name.replace("_", " ") # Latex fix 

444 conf_id = conf_id.replace("_", " ") # Latex fix 

445 ranking_list.add_item( 

446 pl.UnsafeCommand( 

447 f"textbf{{{solver_name}}} ({conf_id}): {contribution} ({performance})" 

448 ) 

449 ) 

450 

451 # 3. Marginal contribution ranking list actual selector 

452 report.append( 

453 "The following list shows the marginal contribution ranking list for " 

454 "the actual portfolio selector:" 

455 ) 

456 with report.create(pl.Enumerate()) as ranking_list: 

457 for ( 

458 solver_name, 

459 conf_id, 

460 contribution, 

461 performance, 

462 ) in scenario_output.marginal_contribution_actual: 

463 contribution, performance = ( 

464 round(contribution, MAX_DEC), 

465 round(performance, MAX_DEC), 

466 ) 

467 solver_name = solver_name.replace("_", " ") # Latex fix 

468 conf_id = conf_id.replace("_", " ") # Latex fix 

469 ranking_list.add_item( 

470 pl.UnsafeCommand( 

471 f"textbf{{{solver_name}}} ({conf_id}): {contribution} ({performance})" 

472 ) 

473 ) 

474 

475 # 4. Create scatter plot analysis 

476 report.append(pl.Subsubsection("Scatter Plot Analysis")) 

477 report.append(latex.AutoRef(f"fig:sbsvsselector{time_stamp}")) 

478 report.append(pl.utils.bold(" ")) # Trick to force a white space 

479 report.append( 

480 "shows the empirical comparison between the portfolio " 

481 "selector and the single best solver (SBS). " 

482 ) 

483 report.append(latex.AutoRef("fig:vbsvsselector")) 

484 report.append(pl.utils.bold(" ")) # Trick to force a white space 

485 report.append( 

486 "shows the empirical comparison between the actual portfolio selector " 

487 "and the virtual best solver (VBS)." 

488 ) 

489 # Create figure on SBS versus the selector 

490 sbs_name, sbs_config, _ = scenario_output.solver_performance_ranking[0] 

491 # sbs_plot_name = f"{Path(sbs_name).name} ({sbs_config})" 

492 sbs_performance = scenario_output.sbs_performance 

493 selector_performance = scenario_output.actual_performance_data 

494 

495 # Join the data together 

496 

497 df = pd.DataFrame( 

498 [sbs_performance, selector_performance], 

499 index=[f"{Path(sbs_name).name} ({sbs_config})", "Selector"], 

500 dtype=float, 

501 ).T 

502 plot = latex.comparison_plot(df, "Single Best Solver vs Selector") 

503 plot_path = ( 

504 plot_dir / f"{Path(sbs_name).name}_{sbs_config}_vs_" 

505 f"Selector_{scenario.selector.model_class.__name__}.pdf" 

506 ) 

507 plot.write_image(plot_path, width=500, height=500) 

508 with report.create(pl.Figure()) as figure: 

509 figure.add_image( 

510 str(plot_path.relative_to(report_dir)), 

511 width=pl.utils.NoEscape(r"0.6\textwidth"), 

512 ) 

513 figure.add_caption( 

514 "Empirical comparison between the Single Best Solver and the Selector" 

515 ) 

516 label = r"label{fig:sbsvsselector" + str(time_stamp) + r"}" 

517 figure.append(pl.UnsafeCommand(f"{label}")) 

518 

519 # Comparison between the actual portfolio selector in Sparkle and the VBS. 

520 vbs_performance = scenario_output.vbs_performance_data.tolist() 

521 df = pd.DataFrame( 

522 [vbs_performance, selector_performance], 

523 index=["Virtual Best Solver", "Selector"], 

524 dtype=float, 

525 ).T 

526 plot = latex.comparison_plot(df, "Virtual Best Solver vs Selector") 

527 plot_path = ( 

528 plot_dir 

529 / f"Virtual_Best_Solver_vs_Selector_{scenario.selector.model_class.__name__}.pdf" 

530 ) 

531 plot.write_image(plot_path, width=500, height=500) 

532 with report.create(pl.Figure()) as figure: 

533 figure.add_image( 

534 str(plot_path.relative_to(report_dir)), 

535 width=pl.utils.NoEscape(r"0.6\textwidth"), 

536 ) 

537 figure.add_caption( 

538 "Empirical comparison between the Virtual Best Solver and the Selector" 

539 ) 

540 figure.append(pl.UnsafeCommand(r"label{fig:vbsvsselector}")) 

541 

542 if scenario_output.test_sets: 

543 report.append(pl.Subsection("Test Results")) 

544 report.append("The following results are reported on the test set(s):") 

545 with report.create(pl.Itemize()) as latex_list: 

546 for test_set_name, test_set_size in scenario_output.test_sets: 

547 result = round( 

548 scenario_output.test_set_performance[test_set_name], MAX_DEC 

549 ) 

550 latex_list.add_item( 

551 pl.UnsafeCommand( 

552 f"textbf{{{test_set_name}}} ({test_set_size} instances): {result}" 

553 ) 

554 ) 

555 

556 

557def generate_parallel_portfolio_section( 

558 report: pl.Document, scenario: PerformanceDataFrame 

559) -> None: 

560 """Generate a section for a parallel portfolio scenario.""" 

561 report_dir = Path(report.default_filepath).parent 

562 portfolio_name = scenario.csv_filepath.parent.name 

563 time_stamp = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())) 

564 plot_dir = report_dir / f"{portfolio_name.replace(' ', '_')}_plots_{time_stamp}" 

565 plot_dir.mkdir() 

566 report.append(pl.Section(f"Parallel Portfolio {portfolio_name}")) 

567 report.append( 

568 "In this scenario, Sparkle runs the portfolio of Solvers on each instance in " 

569 "parallel with " 

570 f"{gv.settings().parallel_portfolio_num_seeds_per_solver} different " 

571 "seeds. The cutoff time for each solver run is set to " 

572 f"{gv.settings().solver_cutoff_time} seconds." 

573 ) 

574 report.append(pl.Subsection("Solvers & Instance Sets")) 

575 report.append("The following Solvers were used in the portfolio:") 

576 # 1. Report on the Solvers and Instance Sets used for the portfolio 

577 with report.create(pl.Itemize()) as solver_latex_list: 

578 configs = scenario.configurations 

579 for solver in scenario.solvers: 

580 solver_name = solver.replace("_", " ") 

581 solver_latex_list.add_item( 

582 pl.UnsafeCommand( 

583 f"textbf{{{solver_name}}} ({len(configs[solver])} configurations)" 

584 ) 

585 ) 

586 report.append("The following Instance Sets were used in the portfolio:") 

587 instance_sets = set(set_name for set_name, _ in scenario.instance_pairs) 

588 instance_set_count = [ 

589 sum(1 for set_name, _ in scenario.instance_pairs if set_name == instance_set) 

590 for instance_set in instance_sets 

591 ] 

592 with report.create(pl.Itemize()) as instance_set_latex_list: 

593 for set_name, set_size in zip(instance_sets, instance_set_count): 

594 set_name = set_name.replace("_", " ") # Latex fix 

595 instance_set_latex_list.add_item( 

596 pl.UnsafeCommand(f"textbf{{{set_name}}} ({set_size} instances)") 

597 ) 

598 # 2. List which solver was the best on how many instances 

599 report.append(pl.Subsection("Portfolio Performance")) 

600 objective = scenario.objectives[0] 

601 report.append( 

602 f"The objective for the portfolio is {objective}. The " 

603 "following performance of the solvers was found over the instances: " 

604 ) 

605 best_solver_count = {solver: 0 for solver in scenario.solvers} 

606 for instance_pair in scenario.instance_pairs: 

607 ranking = scenario.get_solver_ranking( 

608 objective=objective, instance_pairs=[instance_pair] 

609 ) 

610 best_solver_count[ranking[0][0]] += 1 

611 

612 with report.create(pl.Itemize()) as latex_list: 

613 for solver, count in best_solver_count.items(): 

614 solver_name = solver.replace("_", " ") 

615 latex_list.add_item( 

616 pl.UnsafeCommand( 

617 f"textbf{{{solver_name}}} was the best solver on {count} instance(s)." 

618 ) 

619 ) 

620 # TODO Report how many instances remained unsolved 

621 

622 # 3. Create table showing the performance of the portfolio vs and all solvers, 

623 # by showing the status count and number of times the solver was best 

624 solver_cancelled_count = {solver: 0 for solver in scenario.solvers} 

625 solver_timeout_count = {solver: 0 for solver in scenario.solvers} 

626 status_objective = [ 

627 objective 

628 for objective in scenario.objective_names 

629 if objective.lower().startswith("status") 

630 ][0] 

631 cancelled_status = [ 

632 SolverStatus.UNKNOWN, 

633 SolverStatus.CRASHED, 

634 SolverStatus.WRONG, 

635 SolverStatus.ERROR, 

636 SolverStatus.KILLED, 

637 ] 

638 for solver in scenario.solvers: 

639 status = scenario.get_value(solver=solver, objective=status_objective) 

640 for status in scenario.get_value(solver=solver, objective=status_objective): 

641 status = SolverStatus(status) 

642 if status in cancelled_status: 

643 solver_cancelled_count[solver] += 1 

644 elif status == SolverStatus.TIMEOUT: 

645 solver_timeout_count[solver] += 1 

646 

647 report.append(latex.AutoRef("tab:parallelportfoliotable")) 

648 report.append(pl.utils.bold(" ")) 

649 report.append(" shows the performance of the portfolio on the test set(s).") 

650 tabular = pl.Tabular("r|rrrr") 

651 tabular.add_row(["Solver", objective, "# Timeouts", "# Cancelled", "# Best"]) 

652 tabular.add_hline() 

653 solver_performance = { 

654 solver: round(performance, MAX_DEC) 

655 for solver, _, performance in scenario.get_solver_ranking(objective=objective) 

656 } 

657 for solver in scenario.solvers: 

658 tabular.add_row( 

659 solver, 

660 solver_performance[solver], 

661 solver_timeout_count[solver], 

662 solver_cancelled_count[solver], 

663 best_solver_count[solver], 

664 ) 

665 tabular.add_hline() 

666 portfolio_performance = round( 

667 scenario.best_performance(objective=objective), MAX_DEC 

668 ) 

669 tabular.add_row( 

670 portfolio_name, 

671 portfolio_performance, 

672 sum(solver_timeout_count.values()), 

673 sum(solver_cancelled_count.values()), 

674 sum(best_solver_count.values()), 

675 ) 

676 table_portfolio = pl.Table(position="h") 

677 table_portfolio.append(pl.UnsafeCommand("centering")) 

678 table_portfolio.append(tabular) 

679 table_portfolio.add_caption("Parallel Portfolio Performance") 

680 table_portfolio.append(pl.UnsafeCommand(r"label{tab:parallelportfoliotable}")) 

681 report.append(table_portfolio) 

682 

683 # 4. Create scatter plot analysis between the portfolio and the single best solver 

684 sbs_name = scenario.get_solver_ranking(objective=objective)[0][0] 

685 sbs_instance_performance = scenario.get_value( 

686 solver=sbs_name, objective=objective.name 

687 ) 

688 sbs_name = Path(sbs_name).name 

689 report.append(latex.AutoRef("fig:portfoliovssbs")) 

690 report.append(pl.utils.bold(" ")) 

691 report.append( 

692 " shows the emprical comparison between the portfolio and the single " 

693 f"best solver (SBS) {sbs_name}." 

694 ) 

695 portfolio_instance_performance = scenario.best_instance_performance( 

696 objective=objective.name 

697 ).tolist() 

698 

699 df = pd.DataFrame( 

700 [sbs_instance_performance, portfolio_instance_performance], 

701 index=[f"SBS ({sbs_name}) Performance", "Portfolio Performance"], 

702 dtype=float, 

703 ).T 

704 plot = latex.comparison_plot(df, None) 

705 plot_path = plot_dir / f"sbs_{sbs_name}_vs_parallel_portfolio.pdf" 

706 plot.write_image(plot_path, width=500, height=500) 

707 with report.create(pl.Figure(position="h")) as figure: 

708 figure.add_image( 

709 str(plot_path.relative_to(report_dir)), 

710 width=pl.utils.NoEscape(r"0.6\textwidth"), 

711 ) 

712 figure.add_caption(f"Portfolio vs SBS Performance ({objective})") 

713 figure.append(pl.UnsafeCommand(r"label{fig:portfoliovssbs}")) 

714 

715 

716def append_dataframe_longtable( 

717 report: pl.Document, 

718 df: pd.DataFrame, 

719 caption: str, 

720 label: str, 

721 max_cols: int = MAX_COLS_PER_TABLE, 

722 wide_threshold: int = WIDE_TABLE_THRESHOLD, 

723 num_keys: int = NUM_KEYS_PDF, 

724) -> None: 

725 """Appends a pandas DataFrame to a PyLaTeX document as one or more LaTeX longtables. 

726 

727 Args: 

728 report: The PyLaTeX document to which the table(s) will be appended. 

729 df: The DataFrame to be rendered as LaTeX longtable(s). 

730 caption: The caption for the table(s). 

731 label: The LaTeX label for referencing the table(s). 

732 max_cols: Maximum number of columns per table chunk. 

733 Defaults to MAX_COLS_PER_TABLE. 

734 wide_threshold: Number of columns above which the table is rotated 

735 to landscape. Defaults to WIDE_TABLE_THRESHOLD. 

736 num_keys: Number of key columns to include in each table chunk. 

737 Defaults to NUM_KEYS_PDF. 

738 

739 Returns: 

740 None 

741 """ 

742 import math 

743 from typing import Union 

744 

745 def latex_escape_text(string: str) -> str: 

746 """Escape special LaTeX characters in a string.""" 

747 # escape text, but insert our own LaTeX macro around it 

748 return ( 

749 string.replace("\\", r"\textbackslash{}")