Coverage for sparkle/platform/output/structures.py: 96%

1#!/usr/bin/env python3 

2"""Sparkle output structures.""" 

3from __future__ import annotations 

4import sys 

5from pathlib import Path 

6 

7from runrunner.base import Status 

8 

9from sparkle.solver import Solver 

10from sparkle.types import SolverStatus, SparkleObjective 

11from sparkle.instance import InstanceSet 

12from sparkle.structures import PerformanceDataFrame 

13 

14 

15class ValidationResults: 

16 """Class that stores validation information and results.""" 

17 def __init__(self: ValidationResults, solver: Solver, 

18 configuration: dict, instance_set: InstanceSet, 

19 results: list[list[str, Status, float, float]]) -> None: 

20 """Initalize ValidationResults. 

21 

22 Args: 

23 solver: The name of the solver 

24 configuration: The configuration being used 

25 instance_set: The set of instances 

26 results: Validation results in the format: 

27 [["instance", "status", "quality", "runtime"]] 

28 """ 

29 self.solver = solver 

30 self.configuration = configuration 

31 self.instance_set = instance_set 

32 self.result_header = ["instance", "status", "quality", "runtime"] 

33 self.result_vals = results 

34 

35 

36class ConfigurationResults: 

37 """Class that aggregates configuration results.""" 

38 def __init__(self: ConfigurationResults, metrics: float, 

39 results: ValidationResults) -> None: 

40 """Initalize ConfigurationResults. 

41 

42 Args: 

43 metrics: The performance of a configured solver 

44 results: The results for one configuration 

45 """ 

46 self.performance = metrics 

47 self.results = results 

48 

49 

50class SelectionSolverData: 

51 """Class that stores solver information.""" 

52 def __init__(self: SelectionSolverData, 

53 solver_performance_ranking: list[tuple[str, float]], 

54 num_solvers: int) -> None: 

55 """Initalize SelectionSolverData. 

56 

57 Args: 

58 solver_performance_ranking: list with solvers ranked by avg. performance 

59 num_solvers: The number of solvers 

60 """ 

61 self.solver_performance_ranking = solver_performance_ranking 

62 self.single_best_solver = solver_performance_ranking[0][0] 

63 self.num_solvers = num_solvers 

64 

65 

66class SelectionPerformance: 

67 """Class that stores selection performance results.""" 

68 def __init__(self: SelectionSolverData, 

69 performance_path: Path, 

70 vbs_performance: float, 

71 objective: SparkleObjective) -> None: 

72 """Initalize SelectionPerformance. 

73 

74 Args: 

75 performance_path: Path to portfolio selector performance 

76 vbs_performance: The performance of the virtual best selector 

77 objective: The objective (Performance type) 

78 """ 

79 if not performance_path.exists(): 

80 print(f"ERROR: {performance_path} does not exist.") 

81 sys.exit(-1) 

82 actual_performance_data = PerformanceDataFrame(performance_path) 

83 self.vbs_performance = vbs_performance 

84 self.actual_performance = actual_performance_data.mean( 

85 objective=objective.name) 

86 self.metric = objective.name 

87 

88 

89class ParallelPortfolioResults: 

90 """Class that stores parallel portfolio results.""" 

91 def __init__(self: ParallelPortfolioResults, 

92 unsolved_instances: int, 

93 sbs: str, 

94 runtime_solvers: dict[str, float], 

95 instance_results: dict[str, list]) -> None: 

96 """Initalize SelectionSolverData. 

97 

98 Args: 

99 unsolved_instances: Number of unsolved instances 

100 sbs: Name of the single best solver 

101 runtime_solvers: Dictionary containing penalised average runtime per solver 

102 instance_results: Dictionary containing 

103 """ 

104 self.unsolved_instances = unsolved_instances 

105 self.sbs = sbs 

106 self.runtime_solvers = runtime_solvers 

107 self.instance_results = instance_results 

108 

109 self.solver_performance = {} 

110 # Iterate over each instance and aggregate the results 

111 for _, results in self.instance_results.items(): 

112 for solver_result in results: 

113 solver_name = solver_result[0] 

114 outcome = solver_result[1] 

115 # Initialize the solver's record in solver_performance if not present 

116 if solver_name not in self.solver_performance: 

117 self.solver_performance[solver_name] = { 

118 status: 0 for status in SolverStatus 

119 } 

120 # Increment the appropriate outcome count 

121 self.solver_performance[solver_name][outcome] += 1