Coverage for sparkle/platform/output/configuration_output.py: 20%
55 statements
« prev ^ index » next coverage.py v7.9.1, created at 2025-07-01 13:21 +0000
1"""Sparkle class to organise configuration output."""
2from __future__ import annotations
4from sparkle.structures import PerformanceDataFrame
5from sparkle.instance import InstanceSet
6from sparkle.configurator.configurator import ConfigurationScenario
7from sparkle.types import SparkleObjective, SolverStatus
10class ConfigurationResult:
11 """Class that represents result of configuration on an instance set."""
13 def __init__(self: ConfigurationResult,
14 instance_set: str,
15 default_instance_performance: list[float],
16 best_instance_performance: list[float],
17 instance_status_default: dict[str, SolverStatus],
18 instance_status_best: dict[str, SolverStatus],
19 objective: SparkleObjective) -> None:
20 """Initialize a configuration result.
22 All input sequences are the results per instance.
24 Args:
25 instance_set: The name of the instance set
26 default_instance_performance: The default instance performance
27 best_instance_performance: The best instance performance
28 performance: The performance of the configuration
29 instance_status_default: The status of the default configuration
30 instance_status_best: The status of the best configuration
31 objective: The objective
32 """
33 self.default_instance_performance = default_instance_performance
34 self.default_performance: float = objective.instance_aggregator(
35 default_instance_performance)
36 self.best_instance_performance = best_instance_performance
37 self.best_performance: float = objective.instance_aggregator(
38 best_instance_performance)
39 self.instance_status_default = instance_status_default
40 self.instance_status_best = instance_status_best
41 self.instance_set_name = instance_set
43 def serialise(self: ConfigurationResult) -> dict[str, float | list[float]]:
44 """Serialise the data."""
45 return {
46 "instance_set": self.instance_set_name,
47 "default_performance": self.default_performance,
48 "best_performance": self.best_performance,
49 "default_instance_performance": self.default_instance_performance,
50 "best_instance_performance": self.best_instance_performance,
51 "instance_status_default": self.instance_status_default,
52 "instance_status_best": self.instance_status_best
53 }
56class ConfigurationOutput:
57 """Class that collects configuration data and outputs it a JSON format."""
59 def __init__(self: ConfigurationOutput,
60 config_scenario: ConfigurationScenario,
61 performance_data: PerformanceDataFrame,
62 possible_test_sets: list[InstanceSet] = None,
63 ) -> None:
64 """Initialize Configurator Output class.
66 Args:
67 config_scenario: The scenario to output
68 performance_data: Performance data
69 possible_test_sets: Instance Sets possibly used for testing
70 """
71 self.solver = config_scenario.solver
72 self.configurator = config_scenario.configurator
73 self.instance_set_train = config_scenario.instance_set
75 # Filter data on this scenario
76 performance_data_config = performance_data.clone()
77 performance_data_config.remove_solver([s for s in performance_data_config.solvers
78 if s != str(self.solver.directory)])
79 used_configs = config_scenario.configuration_ids + [
80 PerformanceDataFrame.default_configuration]
81 removable = [c for c in performance_data_config.configuration_ids
82 if c not in used_configs]
83 performance_data_config.remove_configuration(
84 str(self.solver.directory), removable)
85 self.test_instance_sets = []
86 for test_set in possible_test_sets:
87 if test_set.name == self.instance_set_train.name:
88 continue
89 for instance in test_set.instance_names:
90 if instance not in performance_data_config.instances or\
91 performance_data_config.is_missing(
92 str(self.solver.directory), instance):
93 continue
94 self.test_instance_sets.append(test_set)
95 self.directory = config_scenario.directory
96 self.config_scenario = config_scenario
98 # Retrieve all configurations
99 solver_key = str(self.solver.directory)
100 config_keys = performance_data_config.get_configurations(solver_key)
101 self.all_configurations = performance_data_config.get_full_configuration(
102 solver_key, config_keys)
104 # Retrieve configuration performances
105 train_instances = self.instance_set_train.instance_names
106 # Retrieve Default (No configuration) performance
107 _, self.default_performance_train =\
108 performance_data_config.configuration_performance(
109 solver_key, PerformanceDataFrame.default_configuration,
110 objective=self.config_scenario.sparkle_objectives[0],
111 instances=train_instances)
113 _, self.default_performance_per_instance_train =\
114 performance_data_config.configuration_performance(
115 solver_key, PerformanceDataFrame.default_configuration,
116 objective=self.config_scenario.sparkle_objectives[0],
117 instances=train_instances,
118 per_instance=True)
120 # Retrieve best found configuration
121 self.best_configuration_key, self.best_performance_train =\
122 performance_data_config.best_configuration(
123 solver_key,
124 objective=self.config_scenario.sparkle_objective,
125 instances=train_instances)
126 self.best_configuration = self.all_configurations[config_keys.index(
127 self.best_configuration_key)]
129 # TODO keep all instance set performance data together in a dictionary instead
130 # of variables for train and test
131 # Shitty hack to get status objective
132 status_objective = [o for o in performance_data_config.objective_names
133 if o.lower().startswith("status")][0]
134 self.instance_set_results: dict[str, ConfigurationResult] = {}
135 for instance_set in self.test_instance_sets + [self.instance_set_train]:
136 instances = instance_set.instance_names
137 _, default_performance_per_instance =\
138 performance_data_config.configuration_performance(
139 solver_key, PerformanceDataFrame.default_configuration,
140 objective=self.config_scenario.sparkle_objective,
141 instances=instances,
142 per_instance=True)
143 _, best_conf_performance_per_instance =\
144 performance_data_config.configuration_performance(
145 solver_key, self.best_configuration_key,
146 objective=self.config_scenario.sparkle_objective,
147 instances=instances,
148 per_instance=True)
149 instance_status_default = {str(i): performance_data_config.get_value(
150 solver_key,
151 configuration=PerformanceDataFrame.default_configuration,
152 objective=status_objective,
153 instance=[i]) for i in instances}
154 instance_status_best_conf = {str(i): performance_data_config.get_value(
155 solver_key,
156 configuration=self.best_configuration_key,
157 objective=status_objective,
158 instance=[i]) for i in instances}
159 self.instance_set_results[instance_set.name] = ConfigurationResult(
160 instance_set.name,
161 default_performance_per_instance,
162 best_conf_performance_per_instance,
163 instance_status_default,
164 instance_status_best_conf,
165 self.config_scenario.sparkle_objectives[0]
166 )
168 def serialise(self: ConfigurationOutput) -> dict:
169 """Serialise the configuration output."""
170 return {
171 "solver": self.solver.name,
172 "configurator": self.configurator.__name__,
173 "best_configuration": self.best_configuration,
174 "best_performance_train": self.best_performance_train,
175 "scenario": self.config_scenario.serialise(),
176 }