Coverage for sparkle/platform/output/configuration_output.py: 20%
55 statements
« prev ^ index » next coverage.py v7.10.7, created at 2025-09-29 10:17 +0000
1"""Sparkle class to organise configuration output."""
3from __future__ import annotations
5from sparkle.structures import PerformanceDataFrame
6from sparkle.instance import InstanceSet
7from sparkle.configurator.configurator import ConfigurationScenario
8from sparkle.types import SparkleObjective, SolverStatus
11class ConfigurationResult:
12 """Class that represents result of configuration on an instance set."""
14 def __init__(
15 self: ConfigurationResult,
16 instance_set: str,
17 default_instance_performance: list[float],
18 best_instance_performance: list[float],
19 instance_status_default: dict[str, SolverStatus],
20 instance_status_best: dict[str, SolverStatus],
21 objective: SparkleObjective,
22 ) -> None:
23 """Initialize a configuration result.
25 All input sequences are the results per instance.
27 Args:
28 instance_set: The name of the instance set
29 default_instance_performance: The default instance performance
30 best_instance_performance: The best instance performance
31 performance: The performance of the configuration
32 instance_status_default: The status of the default configuration
33 instance_status_best: The status of the best configuration
34 objective: The objective
35 """
36 self.default_instance_performance = default_instance_performance
37 self.default_performance: float = objective.instance_aggregator(
38 default_instance_performance
39 )
40 self.best_instance_performance = best_instance_performance
41 self.best_performance: float = objective.instance_aggregator(
42 best_instance_performance
43 )
44 self.instance_status_default = instance_status_default
45 self.instance_status_best = instance_status_best
46 self.instance_set_name = instance_set
48 def serialise(self: ConfigurationResult) -> dict[str, float | list[float]]:
49 """Serialise the data."""
50 return {
51 "instance_set": self.instance_set_name,
52 "default_performance": self.default_performance,
53 "best_performance": self.best_performance,
54 "default_instance_performance": self.default_instance_performance,
55 "best_instance_performance": self.best_instance_performance,
56 "instance_status_default": self.instance_status_default,
57 "instance_status_best": self.instance_status_best,
58 }
61class ConfigurationOutput:
62 """Class that collects configuration data and outputs it a JSON format."""
64 def __init__(
65 self: ConfigurationOutput,
66 config_scenario: ConfigurationScenario,
67 performance_data: PerformanceDataFrame,
68 possible_test_sets: list[InstanceSet] = None,
69 ) -> None:
70 """Initialize Configurator Output class.
72 Args:
73 config_scenario: The scenario to output
74 performance_data: Performance data
75 possible_test_sets: Instance Sets possibly used for testing
76 """
77 self.solver = config_scenario.solver
78 self.configurator = config_scenario.configurator
79 self.instance_set_train = config_scenario.instance_set
81 # Filter data on this scenario
82 performance_data_config = performance_data.clone()
83 performance_data_config.remove_solver(
84 [
85 s
86 for s in performance_data_config.solvers
87 if s != str(self.solver.directory)
88 ]
89 )
90 used_configs = config_scenario.configuration_ids + [
91 PerformanceDataFrame.default_configuration
92 ]
93 removable = [
94 c for c in performance_data_config.configuration_ids if c not in used_configs
95 ]
96 performance_data_config.remove_configuration(
97 str(self.solver.directory), removable
98 )
99 self.test_instance_sets = []
100 for test_set in possible_test_sets:
101 if test_set.name == self.instance_set_train.name:
102 continue
103 for instance in test_set.instance_names:
104 if (
105 instance not in performance_data_config.instances
106 or performance_data_config.is_missing(
107 str(self.solver.directory), instance
108 )
109 ):
110 continue
111 self.test_instance_sets.append(test_set)
112 self.directory = config_scenario.directory
113 self.config_scenario = config_scenario
115 # Retrieve all configurations
116 solver_key = str(self.solver.directory)
117 config_keys = performance_data_config.get_configurations(solver_key)
118 self.all_configurations = performance_data_config.get_full_configuration(
119 solver_key, config_keys
120 )
122 # Retrieve configuration performances
123 train_instances = self.instance_set_train.instance_names
124 # Retrieve Default (No configuration) performance
125 _, self.default_performance_train = (
126 performance_data_config.configuration_performance(
127 solver_key,
128 PerformanceDataFrame.default_configuration,
129 objective=self.config_scenario.sparkle_objectives[0],
130 instances=train_instances,
131 )
132 )
134 _, self.default_performance_per_instance_train = (
135 performance_data_config.configuration_performance(
136 solver_key,
137 PerformanceDataFrame.default_configuration,
138 objective=self.config_scenario.sparkle_objectives[0],
139 instances=train_instances,
140 per_instance=True,
141 )
142 )
144 # Retrieve best found configuration
145 self.best_configuration_key, self.best_performance_train = (
146 performance_data_config.best_configuration(
147 solver_key,
148 objective=self.config_scenario.sparkle_objective,
149 instances=train_instances,
150 )
151 )
152 self.best_configuration = self.all_configurations[
153 config_keys.index(self.best_configuration_key)
154 ]
156 # TODO keep all instance set performance data together in a dictionary instead
157 # of variables for train and test
158 # Shitty hack to get status objective
159 status_objective = [
160 o
161 for o in performance_data_config.objective_names
162 if o.lower().startswith("status")
163 ][0]
164 self.instance_set_results: dict[str, ConfigurationResult] = {}
165 for instance_set in self.test_instance_sets + [self.instance_set_train]:
166 instances = instance_set.instance_names
167 _, default_performance_per_instance = (
168 performance_data_config.configuration_performance(
169 solver_key,
170 PerformanceDataFrame.default_configuration,
171 objective=self.config_scenario.sparkle_objective,
172 instances=instances,
173 per_instance=True,
174 )
175 )
176 _, best_conf_performance_per_instance = (
177 performance_data_config.configuration_performance(
178 solver_key,
179 self.best_configuration_key,
180 objective=self.config_scenario.sparkle_objective,
181 instances=instances,
182 per_instance=True,
183 )
184 )
185 instance_status_default = {
186 str(i): performance_data_config.get_value(
187 solver_key,
188 configuration=PerformanceDataFrame.default_configuration,
189 objective=status_objective,
190 instance=[i],
191 )
192 for i in instances
193 }
194 instance_status_best_conf = {
195 str(i): performance_data_config.get_value(
196 solver_key,
197 configuration=self.best_configuration_key,
198 objective=status_objective,
199 instance=[i],
200 )
201 for i in instances
202 }
203 self.instance_set_results[instance_set.name] = ConfigurationResult(
204 instance_set.name,
205 default_performance_per_instance,
206 best_conf_performance_per_instance,
207 instance_status_default,
208 instance_status_best_conf,
209 self.config_scenario.sparkle_objectives[0],
210 )
212 def serialise(self: ConfigurationOutput) -> dict:
213 """Serialise the configuration output."""
214 return {
215 "solver": self.solver.name,
216 "configurator": self.configurator.__name__,
217 "best_configuration": self.best_configuration,
218 "best_performance_train": self.best_performance_train,
219 "scenario": {
220 str(key): str(value)
221 for key, value in self.config_scenario.serialise().items()
222 },
223 }