Coverage for src/sparkle/structures/performance_dataframe.py: 84%
474 statements
« prev ^ index » next coverage.py v7.16.0, created at 2026-09-08 12:00 +0000
1"""Module to manage performance data files and common operations on them."""
3from __future__ import annotations
4import ast
5import copy
6from typing import Any
7import itertools
8from pathlib import Path
9import math
10import numpy as np
11import pandas as pd
13from sparkle.types import SparkleObjective, resolve_objective
16class PerformanceDataFrame(pd.DataFrame):
17 """Class to manage performance data and common operations on them."""
19 missing_value = math.nan
21 missing_objective = "UNKNOWN"
22 default_configuration = "Default"
24 index_objective = "Objective"
25 index_instance_set = "InstanceSet"
26 index_instance = "Instance"
27 index_run = "Run"
28 multi_index_names = [index_objective, index_instance_set, index_instance, index_run]
30 column_solver = "Solver"
31 column_configuration = "Configuration"
32 column_meta = "Meta"
33 column_value = "Value"
34 column_seed = "Seed"
35 multi_column_names = [column_solver, column_configuration, column_meta]
36 multi_column_value = [column_value, column_seed]
37 multi_column_dtypes = [str, int]
39 def __init__(
40 self: PerformanceDataFrame,
41 csv_filepath: Path,
42 solvers: list[str] = None,
43 configurations: dict[str, dict[str, dict]] = None,
44 objectives: list[str | SparkleObjective] = None,
45 instance_pairs: list[tuple[str, str]] = None,
46 n_runs: int = 1,
47 ) -> None:
48 """Initialise a PerformanceDataFrame.
50 Consists of:
51 - Columns representing the Solvers
52 - Rows representing the result by multi-index in order of:
53 * Objective (Static, given in constructor or read from file)
54 * InstanceSet
55 * Instance
56 * Runs (Static, given in constructor or read from file)
58 Args:
59 csv_filepath: If path exists, load from Path.
60 Otherwise create new and save to this path.
61 solvers: List of solver names to be added into the Dataframe
62 configurations: The configuration keys per solver to add, structured as
63 configurations[solver][config_key] = {"parameter": "value", ..}
64 objectives: List of SparkleObjectives or objective names. By default None,
65 then the objectives will be derived from Sparkle Settings if possible.
66 instance_pairs: List of (set_name, instance_name) pairs to add. By default None.
67 n_runs: The number of runs to consider per Solver/Objective/Instance comb.
68 """
69 if csv_filepath and csv_filepath.exists(): # Read from file
70 df = pd.read_csv(
71 csv_filepath,
72 header=[0, 1, 2],
73 index_col=[0, 1, 2, 3],
74 on_bad_lines="skip",
75 dtype={
76 PerformanceDataFrame.column_value: str,
77 PerformanceDataFrame.column_seed: int,
78 # PerformanceDataFrame.index_run: int, # NOTE: Preferrably, this would be set, but it is not included in the "on_bad_lines=skip" case for error lines.
79 },
80 comment="$",
81 ) # $ For extra data lines
82 super().__init__(df)
83 self.csv_filepath = csv_filepath
84 # Load configuration mapping
85 with self.csv_filepath.open() as file:
86 configuration_lines = [
87 line.strip().strip("$").split(",", maxsplit=2)
88 for line in file.readlines()
89 if line.startswith("$")
90 ]
91 configurations = {solver: {} for solver in self.solvers}
92 for solver, config_key, config in configuration_lines[1:]: # Skip header
93 if (
94 solver in configurations
95 ): # Only add configurations to already known solvers, based on the columns
96 configurations[solver][config_key] = ast.literal_eval(
97 config.strip('"')
98 )
99 else: # New PerformanceDataFrame
100 # Initialize empty DataFrame
101 run_ids = list(range(1, n_runs + 1)) # We count runs from 1
102 # We always need objectives to maintain the dimensions
103 if objectives is None:
104 objectives = [PerformanceDataFrame.missing_objective]
105 else:
106 objectives = [str(objective) for objective in objectives]
107 # We always need an instance to maintain the dimensions
108 if instance_pairs is None:
109 instance_pairs = [
110 (
111 PerformanceDataFrame.missing_value,
112 PerformanceDataFrame.missing_value,
113 )
114 ]
115 # We always need a solver to maintain the dimensions
116 if solvers is None:
117 solvers = [PerformanceDataFrame.missing_value]
118 # Build the 4-level index explicitly (from_product can't handle pair instances)
119 midx = pd.MultiIndex.from_tuples(
120 [
121 (objective, set_name, instance, run)
122 for objective in objectives
123 for (set_name, instance) in instance_pairs
124 for run in run_ids
125 ],
126 names=PerformanceDataFrame.multi_index_names,
127 )
128 # Create the multi index tuples
129 if configurations is None:
130 configurations = {
131 solver: {PerformanceDataFrame.default_configuration: {}}
132 for solver in solvers
133 }
134 column_tuples = []
135 # We cannot do .from_product here as config ids are per solver
136 for solver in configurations.keys():
137 for config_id in configurations[solver].keys():
138 column_tuples.extend(
139 [
140 (solver, config_id, PerformanceDataFrame.column_seed),
141 (solver, config_id, PerformanceDataFrame.column_value),
142 ]
143 )
144 mcolumns = pd.MultiIndex.from_tuples(
145 column_tuples,
146 names=PerformanceDataFrame.multi_column_names,
147 )
148 # Set dtype object to avoid inferring float for categorical objectives
149 super().__init__(
150 PerformanceDataFrame.missing_value,
151 index=midx,
152 columns=mcolumns,
153 dtype="object",
154 )
155 self.csv_filepath = csv_filepath
157 # Store configuration in global attributes dictionary, see Pandas Docs
158 self.attrs = configurations
160 if self.index.duplicated().any(): # Drop all duplicates except for last
161 # NOTE: This is rather convoluted (but fast!) due to the fact we need to do it inplace to maintain our type (PerformanceDataFrame)
162 # Make the index levels into columns (in-place)
163 self.reset_index(inplace=True)
164 # The first nlevels columns are the index columns created by reset_index, drop duplicates in those columns
165 idx_cols = self.columns[
166 : len(PerformanceDataFrame.multi_index_names)
167 ].tolist()
168 self.drop_duplicates(
169 subset=idx_cols, keep="last", inplace=True
170 ) # Drop duplicates
171 self.set_index(idx_cols, inplace=True) # Restore the MultiIndex (in-place)
172 self.index.rename(
173 PerformanceDataFrame.multi_index_names, inplace=True
174 ) # Restore level names
176 # Sort the index to optimize lookup speed
177 self.sort_index(axis=0, inplace=True)
178 self.sort_index(axis=1, inplace=True)
180 if csv_filepath and not self.csv_filepath.exists(): # New Performance DataFrame
181 self.save_csv()
183 # Properties
185 @property
186 def num_objectives(self: PerformanceDataFrame) -> int:
187 """Retrieve the number of objectives in the DataFrame."""
188 return (
189 self.index.get_level_values(PerformanceDataFrame.index_objective)
190 .unique()
191 .size
192 )
194 @property
195 def num_instances(self: PerformanceDataFrame) -> int:
196 """Return the number of unique (InstanceSet, Instance) pairs."""
197 return len(self.instance_pairs)
199 @property
200 def num_runs(self: PerformanceDataFrame) -> int:
201 """Return the maximum number of runs of each instance."""
202 return self.index.get_level_values(PerformanceDataFrame.index_run).unique().size
204 @property
205 def num_solvers(self: PerformanceDataFrame) -> int:
206 """Return the number of solvers."""
207 return self.columns.get_level_values(0).unique().size
209 @property
210 def num_solver_configurations(self: PerformanceDataFrame) -> int:
211 """Return the number of solver configurations."""
212 return int(
213 self.columns.get_level_values( # Config has a seed & value
214 PerformanceDataFrame.column_configuration
215 ).size
216 / 2
217 )
219 @property
220 def multi_objective(self: PerformanceDataFrame) -> bool:
221 """Return whether the dataframe represent MO or not."""
222 return self.num_objectives > 1
224 @property
225 def solvers(self: PerformanceDataFrame) -> list[str]:
226 """Return the solver present as a list of strings."""
227 # Do not return the nan solver as its not an actual solver
228 return (
229 self.columns.get_level_values(PerformanceDataFrame.column_solver)
230 .dropna()
231 .unique()
232 .to_list()
233 )
235 @property
236 def configuration_ids(self: PerformanceDataFrame) -> list[str]:
237 """Return the list of configuration keys."""
238 return (
239 self.columns.get_level_values(PerformanceDataFrame.column_configuration)
240 .unique()
241 .to_list()
242 )
244 @property
245 def configurations(self: PerformanceDataFrame) -> dict[str, dict[str, dict]]:
246 """Return a dictionary (copy) containing the configurations for each solver."""
247 return copy.deepcopy(self.attrs) # Deepcopy to avoid mutation of attribute
249 @property
250 def objective_names(self: PerformanceDataFrame) -> list[str]:
251 """Return the objective names as a list of strings."""
252 return (
253 self.index.get_level_values(PerformanceDataFrame.index_objective)
254 .unique()
255 .to_list()
256 )
258 @property
259 def objectives(self: PerformanceDataFrame) -> list[SparkleObjective]:
260 """Return the objectives as a list of SparkleObjectives."""
261 return [resolve_objective(objective) for objective in self.objective_names]
263 @property
264 def instance_pairs(self: PerformanceDataFrame) -> list[tuple[str, str]]:
265 """Return the (set_name, instance_name) pairs as a list."""
266 set_vals = self.index.get_level_values(PerformanceDataFrame.index_instance_set)
267 inst_vals = self.index.get_level_values(PerformanceDataFrame.index_instance)
268 pairs = list(zip(set_vals, inst_vals))
269 # Unique, order-preserving
270 return list(dict.fromkeys(pairs))
272 @property
273 def instance_sets(self: PerformanceDataFrame) -> list[str]:
274 """Return the unique instance set names."""
275 return (
276 self.index.get_level_values(PerformanceDataFrame.index_instance_set)
277 .unique()
278 .tolist()
279 )
281 @property
282 def run_ids(self: PerformanceDataFrame) -> list[int]:
283 """Return the run ids as a list of integers."""
284 return (
285 self.index.get_level_values(PerformanceDataFrame.index_run)
286 .unique()
287 .to_list()
288 )
290 @property
291 def has_missing_values(self: PerformanceDataFrame) -> bool:
292 """Returns True if there are any missing values in the dataframe."""
293 return (
294 self.drop(
295 PerformanceDataFrame.column_seed,
296 level=PerformanceDataFrame.column_meta,
297 axis=1,
298 )
299 .isnull()
300 .any()
301 .any()
302 )
304 def is_missing(
305 self: PerformanceDataFrame,
306 solver: str,
307 instance_set: str,
308 instance_name: str,
309 ) -> int:
310 """Check whether a solver has any missing values for an instance.
312 Args:
313 solver: Solver to be checked.
314 instance_set: The name of the set the instance belongs to.
315 instance_name: The name of the instance.
317 Returns:
318 True(1) if any value (excluding the seed) is missing for the given
319 solver/instance combination across all objectives, configurations
320 and runs, False otherwise.
321 """
322 return (
323 self.xs(solver, axis=1)
324 .xs(instance_set, axis=0, level=PerformanceDataFrame.index_instance_set)
325 .xs(instance_name, axis=0, level=PerformanceDataFrame.index_instance)
326 .drop(
327 PerformanceDataFrame.column_seed,
328 level=PerformanceDataFrame.column_meta,
329 axis=1,
330 )
331 .isnull()
332 .any()
333 .any()
334 )
336 def verify_objective(self: PerformanceDataFrame, objective: str) -> str:
337 """Method to check whether the specified objective is valid.
339 Users are allowed to index the dataframe without specifying all dimensions.
340 However, when dealing with multiple objectives this is not allowed and this
341 is verified here. If we have only one objective this is returned. Otherwise,
342 if an objective is specified by the user this is returned.
344 Args:
345 objective: The objective given by the user
346 """
347 if objective is None:
348 if self.multi_objective:
349 raise ValueError("Error: MO Data, but objective not specified.")
350 elif self.num_objectives == 1:
351 return self.objective_names[0]
352 else:
353 return PerformanceDataFrame.missing_objective
354 return objective
356 def verify_run_id(self: PerformanceDataFrame, run_id: int) -> int:
357 """Method to check whether run id is valid.
359 Similar to verify_objective but here we check the dimensionality of runs.
361 Args:
362 run_id: the run as specified by the user.
363 """
364 if run_id is None:
365 if self.num_runs > 1:
366 raise ValueError(
367 "Error: Multiple run performance data, but run not specified"
368 )
369 else:
370 run_id = self.run_ids[0]
371 return run_id
373 def verify_indexing(
374 self: PerformanceDataFrame, objective: str, run_id: int
375 ) -> tuple[str, int]:
376 """Method to check whether data indexing is correct.
378 Users are allowed to use the Performance Dataframe without the second and
379 fourth dimension (Objective and Run respectively) in the case they only
380 have one objective or only do one run. This method adjusts the indexing for
381 those cases accordingly.
383 Args:
384 objective: The given objective name
385 run_id: The given run index
387 Returns:
388 A tuple representing the (possibly adjusted) Objective and Run index.
389 """
390 objective = self.verify_objective(objective)
391 run_id = self.verify_run_id(run_id)
392 return objective, run_id
394 # Getters and Setters
396 def add_solver(
397 self: PerformanceDataFrame,
398 solver_name: str,
399 configurations: list[(str, dict)] = None,
400 initial_value: float | list[str | float] = None,
401 ) -> None:
402 """Add a new solver to the dataframe. Initializes value to None by default.
404 Args:
405 solver_name: The name of the solver to be added.
406 configurations: A list of configuration keys for the solver.
407 initial_value: The value assigned for each index of the new solver.
408 If not None, must match the index dimension (n_obj * n_inst * n_runs).
409 """
410 if solver_name in self.solvers:
411 print(
412 f"WARNING: Tried adding already existing solver {solver_name} to "
413 f"Performance DataFrame: {self.csv_filepath}"
414 )
415 return
416 if not isinstance(initial_value, list): # Single value
417 initial_value = [[initial_value, initial_value]]
418 if configurations is None:
419 configurations = [(PerformanceDataFrame.default_configuration, {})]
420 self.attrs[solver_name] = {}
421 for (config_key, config), (value, seed) in itertools.product(
422 configurations, initial_value
423 ):
424 self[(solver_name, config_key, PerformanceDataFrame.column_seed)] = seed
425 self[(solver_name, config_key, PerformanceDataFrame.column_value)] = value
426 self.attrs[solver_name][config_key] = config
427 if self.num_solvers == 2: # Remove nan solver
428 for solver in self.solvers:
429 if str(solver) == str(PerformanceDataFrame.missing_value):
430 self.remove_solver(solver)
431 break
433 def add_configuration(
434 self: PerformanceDataFrame,
435 solver: str,
436 configuration_id: str | list[str],
437 configuration: dict[str, Any] | list[dict[str, Any]] = None,
438 ) -> None:
439 """Add new configurations for a solver to the dataframe.
441 If the key already exists, update the value.
443 Args:
444 solver: The name of the solver to be added.
445 configuration_id: The name of the configuration to be added.
446 configuration: The configuration to be added.
447 """
448 if not isinstance(configuration_id, list):
449 configuration_id = [configuration_id]
450 if not isinstance(configuration, list):
451 configuration = [configuration]
452 for config_id, config in zip(configuration_id, configuration):
453 if config_id not in self.get_configurations(solver):
454 self[(solver, config_id, PerformanceDataFrame.column_value)] = None
455 self[(solver, config_id, PerformanceDataFrame.column_seed)] = None
456 self.attrs[solver][config_id] = config
457 # Sort the index to optimize lookup speed
458 self.sort_index(axis=1, inplace=True)
460 def add_objective(
461 self: PerformanceDataFrame, objective_name: str, initial_value: float = None
462 ) -> None:
463 """Add an objective to the DataFrame."""
464 initial_value = initial_value or self.missing_value
465 if objective_name in self.objective_names:
466 print(
467 f"WARNING: Tried adding already existing objective {objective_name} "
468 f"to Performance DataFrame: {self.csv_filepath}"
469 )
470 return
471 for instance_pair, run in itertools.product(self.instance_pairs, self.run_ids):
472 self.loc[(objective_name,) + instance_pair + (run,)] = initial_value
473 self.sort_index(axis=0, inplace=True)
475 def add_instance(
476 self: PerformanceDataFrame,
477 instance_pair: tuple[str, str] | list[tuple[str, str]],
478 initial_values: Any | list[Any] = None,
479 ) -> None:
480 """Add one or more instances to the DataFrame.
482 Args:
483 instance_pair: A (set_name, instance_name) pair, or a list of such pairs
484 to add multiple instances at once.
485 initial_values: The values assigned for each index of the new instance(s).
486 The same values are used for every added instance. If a list, it must
487 match the column dimension (Value, Seed, Configuration).
488 """
489 # Normalise to a list of pairs so the index is built and sorted only once.
490 instance_pairs = (
491 [instance_pair] if isinstance(instance_pair, tuple) else instance_pair
492 )
493 # Normalise initial_values into a full row once; it is shared by every instance.
494 initial_values = initial_values or self.missing_value
495 if not isinstance(initial_values, list):
496 initial_values = (
497 [initial_values]
498 * 2 # Value and Seed per target column
499 * self.num_solver_configurations
500 )
501 elif len(initial_values) == len(PerformanceDataFrame.multi_column_names):
502 initial_values = initial_values * self.num_solvers
504 existing_pairs = set(self.instance_pairs)
505 for instance_pair in instance_pairs:
506 if instance_pair in existing_pairs:
507 print(
508 f"WARNING: Tried adding already existing instance {instance_pair} "
509 f"to Performance DataFrame: {self.csv_filepath}"
510 )
511 continue
512 existing_pairs.add(instance_pair) # Guard against duplicates in the input
513 # Add rows for all combinations
514 for objective, run in itertools.product(self.objective_names, self.run_ids):
515 self.loc[(objective,) + instance_pair + (run,)] = initial_values
517 # Remove the placeholder nan instance now that real instances exist.
518 if self.num_instances > 1:
519 for inst_pair in self.instance_pairs:
520 instance_set, instance = inst_pair
521 if not isinstance(instance, str) and math.isnan(float(instance)):
522 self.remove_instance(inst_pair)
523 break
524 # Sort the index once to optimize lookup speed
525 self.sort_index(axis=0, inplace=True)
527 def add_runs(
528 self: PerformanceDataFrame,
529 num_extra_runs: int,
530 instance_pairs: list[tuple[str, str]] = None,
531 initial_values: Any | list[Any] = None,
532 ) -> None:
533 """Add runs to the DataFrame.
535 Args:
536 num_extra_runs: The number of runs to be added.
537 instance_pairs: The instances for which runs are to be added.
538 By default None, which means runs are added to all instances.
539 initial_values: The initial value for each objective of each new run.
540 If a list, needs to have a value for Value, Seed and Configuration.
541 """
542 initial_values = initial_values or self.missing_value
543 if not isinstance(initial_values, list):
544 initial_values = [initial_values] * self.num_solvers * 2 # Value and Seed
545 elif len(initial_values) == 2: # Value and seed provided
546 initial_values = initial_values * self.num_solvers
547 instance_pairs = (
548 self.instance_pairs if instance_pairs is None else instance_pairs
549 )
550 for objective, instance_pair in itertools.product(
551 self.objective_names, instance_pairs
552 ):
553 index_runs_start = len(self.loc[(objective,) + instance_pair]) + 1
554 for run in range(index_runs_start, index_runs_start + num_extra_runs):
555 self.loc[(objective,) + instance_pair + (run,)] = initial_values
556 # Sort the index to optimize lookup speed
557 # NOTE: It would be better to do this at the end, but that results in
558 # PerformanceWarning: indexing past lexsort depth may impact performance.
559 self.sort_index(axis=0, inplace=True)
561 def get_configurations(self: PerformanceDataFrame, solver_name: str) -> list[str]:
562 """Return the list of configuration keys for a solver."""
563 return list(
564 self[solver_name]
565 .columns.get_level_values(PerformanceDataFrame.column_configuration)
566 .unique()
567 )
569 def get_full_configuration(
570 self: PerformanceDataFrame, solver: str, configuration_id: str | list[str]
571 ) -> dict | list[dict]:
572 """Return the actual configuration associated with the configuration key."""
573 if isinstance(configuration_id, str):
574 return self.attrs[solver][configuration_id]
575 return [self.attrs[solver][cid] for cid in configuration_id]
577 def remove_solver(self: PerformanceDataFrame, solvers: str | list[str]) -> None:
578 """Drop one or more solvers from the Dataframe."""
579 if not solvers: # Bugfix for when an empty list is passed to avoid nan adding
580 return
581 # To make sure objectives / runs are saved when no solvers are present
582 solvers = [solvers] if isinstance(solvers, str) else solvers
583 if self.num_solvers == 1: # This would preferrably be done after removing
584 for field in PerformanceDataFrame.multi_column_value:
585 self[
586 PerformanceDataFrame.missing_value,
587 PerformanceDataFrame.missing_value,
588 field,
589 ] = PerformanceDataFrame.missing_value
590 self.drop(columns=solvers, level=0, axis=1, inplace=True)
591 for solver in solvers:
592 del self.attrs[solver]
594 def remove_configuration(
595 self: PerformanceDataFrame, solver: str, configuration: str | list[str]
596 ) -> None:
597 """Drop one or more configurations from the Dataframe."""
598 if isinstance(configuration, str):
599 configuration = [configuration]
600 for config in configuration:
601 self.drop((solver, config), axis=1, inplace=True)
602 del self.attrs[solver][config]
603 # Sort the index to optimize lookup speed
604 self.sort_index(axis=1, inplace=True)
606 def remove_objective(
607 self: PerformanceDataFrame, objectives: str | list[str]
608 ) -> None:
609 """Remove objective from the Dataframe."""
610 if len(self.objectives) < 2:
611 raise Exception("Cannot remove last objective from PerformanceDataFrame")
612 self.drop(
613 objectives,
614 axis=0,
615 level=PerformanceDataFrame.index_objective,
616 inplace=True,
617 )
619 def remove_instance(
620 self: PerformanceDataFrame,
621 instance_pairs: tuple[str, str] | list[tuple[str, str]],
622 ) -> None:
623 """Drop instances from the Dataframe.
625 Args:
626 instance_pairs: A (set_name, instance_name) pair or list of such pairs.
627 """
628 if not instance_pairs:
629 return
630 if isinstance(instance_pairs, tuple):
631 instance_pairs = [instance_pairs]
632 num_instance_pairs = len(instance_pairs)
633 # To make sure objectives / runs are saved when no instances are present
634 if self.num_instances - num_instance_pairs == 0:
635 for objective, run in itertools.product(self.objective_names, self.run_ids):
636 self.loc[
637 (
638 objective,
639 PerformanceDataFrame.missing_value,
640 PerformanceDataFrame.missing_value,
641 run,
642 )
643 ] = PerformanceDataFrame.missing_value
644 # Build a mask over (InstanceSet, Instance) levels
645 pair_idx = pd.MultiIndex.from_tuples(instance_pairs)
647 # Get the index to be dropped with help of mask
648 to_drop = self.index[
649 self.index.droplevel(
650 [PerformanceDataFrame.index_objective, PerformanceDataFrame.index_run]
651 ).isin(pair_idx)
652 ]
653 self.drop(to_drop, inplace=True)
654 # Sort the index to optimize lookup speed
655 self.sort_index(axis=0, inplace=True)
657 def remove_runs(
658 self: PerformanceDataFrame,
659 runs: int | list[int],
660 instance_pairs: list[tuple[str, str]] = None,
661 ) -> None:
662 """Drop one or more runs from the Dataframe.
664 Args:
665 runs: The run indices to be removed. If its an int,
666 the last n runs are removed. NOTE: If each instance has a different
667 number of runs, the amount of removed runs is not uniform.
668 instance_pairs: The instances for which runs are to be removed.
669 By default None, which means runs are removed from all instances.
670 """
671 instance_pairs = (
672 self.instance_pairs if instance_pairs is None else instance_pairs
673 )
674 runs = (
675 list(range((self.num_runs + 1) - runs, (self.num_runs + 1)))
676 if isinstance(runs, int)
677 else runs
678 )
679 self.drop(runs, axis=0, level=PerformanceDataFrame.index_run, inplace=True)
680 # Sort the index to optimize lookup speed
681 self.sort_index(axis=0, inplace=True)
683 def remove_empty_runs(self: PerformanceDataFrame) -> None:
684 """Remove runs that contain no data, except for the first."""
685 for row_index in self.index:
686 if (
687 row_index[3] == 1
688 ): # Run is at level 3 (Objective, InstanceSet, Instance, Run)
689 continue
690 if self.loc[row_index].isna().all():
691 self.drop(row_index, inplace=True)
693 def filter_objective(self: PerformanceDataFrame, objective: str | list[str]) -> None:
694 """Filter the Dataframe to a subset of objectives."""
695 if isinstance(objective, str):
696 objective = [objective]
697 self.drop(
698 list(set(self.objective_names) - set(objective)),
699 axis=0,
700 level=PerformanceDataFrame.index_objective,
701 inplace=True,
702 )
704 def reset_value(
705 self: PerformanceDataFrame,
706 solver: str,
707 instance_set: str,
708 instance_name: str,
709 objective: str = None,
710 run: int = None,
711 ) -> None:
712 """Reset a value in the dataframe."""
713 self.set_value(
714 PerformanceDataFrame.missing_value,
715 solver,
716 (instance_set, instance_name),
717 objective,
718 run,
719 )
721 def set_value(
722 self: PerformanceDataFrame,
723 value: float | str | list[float | str] | list[list[float | str]],
724 solver: str | list[str],
725 instance_pair: tuple[str, str] | list[tuple[str, str]] | None,
726 configuration: str = None,
727 objective: str | list[str] = None,
728 run: int | list[int] = None,
729 solver_fields: list[str] = ["Value"],
730 append_write_csv: bool = False,
731 ) -> None:
732 """Setter method to assign a value to the Dataframe.
734 Allows for setting the same value to multiple indices.
736 Args:
737 value: Value(s) to be assigned. If value is a list, first dimension is
738 the solver field, second dimension is if multiple different values are
739 to be assigned. Must be the same shape as target.
740 solver: The solver(s) for which the value should be set.
741 If solver is a list, multiple solvers are set. If None, all
742 solvers are set.
743 instance_pair: The (set_name, instance_name) pair for which the value should
744 be set. If None, all instances are set.
745 configuration: The configuration(s) for which the value should be set.
746 When left None, set for all configurations
747 objective: The objectives for which the value should be set.
748 When left None, set for all objectives
749 run: The run index for which the value should be set.
750 If left None, set for all runs.
751 solver_fields: The level to which each value should be assigned.
752 Defaults to ["Value"].
753 append_write_csv: For concurrent writing to the PerformanceDataFrame.
754 If True, the value is directly appended to the CSV file.
755 This will create duplicate entries in the file, but these are combined
756 when loading the file.
757 """
758 # Convert indices to slices for None values
759 solver = solver if solver else slice(solver) # None case
760 configuration = configuration if configuration else slice(configuration)
761 objective = objective if objective else slice(objective)
762 run = run if run else slice(run)
763 if instance_pair is None: # None selects all instances
764 instance_set, instance_name = slice(None), slice(None)
765 elif isinstance(instance_pair, list): # Multiple (set, instance) pairs
766 instance_set = [inst_set for inst_set, inst_name in instance_pair]
767 instance_name = [inst_name for inst_set, inst_name in instance_pair]
768 else: # A single (set, instance) pair
769 instance_set, instance_name = instance_pair
770 row_idx = (objective, instance_set, instance_name, run)
771 # Convert column indices to slices for setting multiple columns
772 value = [value] if not isinstance(value, list) else value
773 # NOTE: We currently forloop levels here, as it allows us to set the same
774 # sequence of values to the indices
775 for item, level in zip(value, solver_fields):
776 self.loc[row_idx, (solver, configuration, level)] = item
778 if append_write_csv:
779 writeable = self.loc[row_idx, :]
780 if isinstance(writeable, pd.Series): # Single row, convert to pd.DataFrame
781 writeable = self.loc[[row_idx], :]
782 # Append the new rows to the dataframe csv file
783 import os
785 csv_string = writeable.to_csv(header=False) # Convert to the csv lines
786 for line in csv_string.splitlines():
787 fd = os.open(f"{self.csv_filepath}", os.O_WRONLY | os.O_APPEND)
788 os.write(fd, f"{line}\n".encode("utf-8")) # Encode to create buffer
789 # Open and close for each line to minimise possibilities of conflict
790 os.close(fd)
792 def get_value(
793 self: PerformanceDataFrame,
794 solver: str | list[str] = None,
795 instance_pair: tuple[str, str] | list[tuple[str, str]] | None = None,
796 configuration: str = None,
797 objective: str = None,
798 run: int = None,
799 solver_fields: list[str] = ["Value"],
800 ) -> float | str | list[Any]:
801 """Index a value of the DataFrame and return it.
803 Any dimension left as None is treated as a wildcard, selecting all
804 entries along that dimension.
806 Args:
807 solver: Solver name or list of solver names. None selects all solvers.
808 instance_pair: A (set_name, instance_name) pair, or None for all instances.
809 configuration: Configuration key to select. None selects all configurations.
810 objective: Objective name to select. None selects all objectives.
811 run: Run id to select. None selects all runs.
812 solver_fields: The solver value fields to return (e.g. "Value", "Seed").
814 Returns:
815 The selected value if a single cell is matched, otherwise a list of
816 the matched values.
817 """
818 # Convert indices to slices for None values
819 solver = solver if solver else slice(solver)
820 configuration = configuration if configuration else slice(configuration)
821 objective = objective if objective else slice(objective)
822 solver_fields = solver_fields if solver_fields else slice(solver_fields)
823 run = run if run else slice(run)
824 if instance_pair is None: # None selects all instances
825 instance_set, instance_name = slice(None), slice(None)
826 elif isinstance(instance_pair, list): # Multiple (set, instance) pairs
827 instance_set = [inst_set for inst_set, inst_name in instance_pair]
828 instance_name = [inst_name for inst_set, inst_name in instance_pair]
829 else: # A single (set, instance) pair
830 instance_set, instance_name = instance_pair
831 row_idx = (objective, instance_set, instance_name, run)
832 target = self.loc[row_idx, (solver, configuration, solver_fields)].values
833 # Reduce dimensions when relevant
834 if len(target) > 0 and isinstance(target[0], np.ndarray) and len(target[0]) == 1:
835 target = target.flatten()
836 target = target.tolist()
837 if len(target) == 1:
838 return target[0]
839 return target
841 def get_instance_num_runs(
842 self: PerformanceDataFrame, instance_set: str, instance_name: str
843 ) -> int:
844 """Return the number of runs for an instance.
846 Args:
847 instance_set: The name of the set the instance belongs to.
848 instance_name: The name of the instance.
849 """
850 # We assume each objective has the same index for Instance/Runs
851 return len(
852 self.loc[(self.objective_names[0], instance_set, instance_name)].index
853 )
855 # Calculables
857 def mean(
858 self: PerformanceDataFrame,
859 objective: str = None,
860 solver: str = None,
861 instance_set: str = None,
862 instance_name: str = None,
863 ) -> float:
864 """Return the mean value of a slice of the dataframe.
866 The slice is narrowed by each provided argument; arguments left as None
867 are not filtered on.
869 Args:
870 objective: Objective to compute the mean over. If None, it is resolved
871 via verify_objective (the sole objective for single objective data).
872 solver: Solver name to restrict the slice to. None includes all solvers.
873 instance_set: Name of the instance set to restrict the slice to. None
874 includes all instance sets.
875 instance_name: Name of the instance to restrict the slice to. None
876 includes all instances.
878 Returns:
879 The mean of all values in the selected slice.
880 """
881 objective = self.verify_objective(objective)
882 subset = self.xs(objective, level=PerformanceDataFrame.index_objective)
883 if solver is not None:
884 subset = subset.xs(solver, axis=1, drop_level=False)
885 # The set name and the instance name live on two separate row levels, so narrow
886 # each level in turn. drop_level=False keeps the remaining MultiIndex levels
887 # intact so the slice still aligns for the .mean() below.
888 if instance_set is not None:
889 subset = subset.xs(
890 instance_set,
891 axis=0,
892 level=PerformanceDataFrame.index_instance_set,
893 drop_level=False,
894 )
895 if instance_name is not None:
896 subset = subset.xs(
897 instance_name,
898 axis=0,
899 level=PerformanceDataFrame.index_instance,
900 drop_level=False,
901 )
902 value = subset.astype(float).mean()
903 if isinstance(value, pd.Series):
904 return value.mean()
905 return value
907 def remaining_jobs(
908 self: PerformanceDataFrame, rerun: bool = False
909 ) -> list[tuple[str, str, tuple[str, str], int]]:
910 """Return a list of performance computation jobs there are to be done.
912 Get a list of jobs to run from the performance data.
913 If rerun is False (default), get only the tuples that don't have a
914 value, else (True) get all the tuples.
916 Args:
917 rerun: Boolean indicating if we want to rerun all jobs
919 Returns:
920 A tuple of (solver, config, (set_name, instance_name), run) combinations
921 """
922 # Drop the seed as we are looking for missing objective values, not seeds.
923 df = self.drop(
924 PerformanceDataFrame.column_seed,
925 axis=1,
926 level=PerformanceDataFrame.column_meta,
927 )
928 df = df.droplevel(PerformanceDataFrame.column_meta, axis=1)
930 # Each job is identified by (instance_set, instance_name, run, solver, config),
931 # independent of objective. Collapse objective level to avoid duplicate generation.
932 if rerun:
933 job_index = df.index.droplevel(PerformanceDataFrame.index_objective).unique()
934 return [
935 (solver, config, (set_name, instance_name), run)
936 for (solver, config), (
937 set_name,
938 instance_name,
939 run,
940 ) in itertools.product(df.columns, job_index)
941 ]
943 # Compute a per-job missingness mask:
944 # True means at least one objective value is still missing.
945 missing_jobs = (
946 df.isna()
947 .groupby(
948 level=[
949 PerformanceDataFrame.index_instance_set,
950 PerformanceDataFrame.index_instance,
951 PerformanceDataFrame.index_run,
952 ],
953 sort=False,
954 )
955 .any()
956 )
957 # Stack the solver and configuration levels to get a MultiIndex of
958 # (instance_set, instance_name, run, solver, config) with boolean missingness.
959 stacked_missing = missing_jobs.stack(
960 [
961 PerformanceDataFrame.column_solver,
962 PerformanceDataFrame.column_configuration,
963 ],
964 future_stack=True,
965 )
967 # Add jobs only when value is True.
968 result = []
969 for (
970 set_name,
971 instance_name,
972 run,
973 solver,
974 config,
975 ), is_missing in stacked_missing.items():
976 if not bool(is_missing):
977 continue
978 # NOTE: Keep historical behavior of skipping invalid run identifiers.
979 if pd.isna(run):
980 continue
981 # NOTE: Force Run to be int, as it can be float on accident.
982 if isinstance(run, (int, float, np.integer, np.floating)):
983 run = int(run)
984 result.append((solver, config, (set_name, instance_name), run))
985 return result
987 def configuration_performance(
988 self: PerformanceDataFrame,
989 solver: str,
990 configuration: str | list[str] = None,
991 objective: str | SparkleObjective = None,
992 instance_pairs: list[tuple[str, str]] = None,
993 per_instance: bool = False,
994 ) -> tuple[str, float]:
995 """Return the (best) configuration performance for objective over the instances.
997 Args:
998 solver: The solver for which we determine evaluate the configuration
999 configuration: The configuration (id) to evaluate
1000 objective: The objective for which we calculate find the best value
1001 instance_pairs: The (set_name, instance_name) pairs to evaluate
1002 per_instance: Whether to return the performance per instance,
1003 or aggregated.
1005 Returns:
1006 The (best) configuration id and its aggregated performance.
1007 """
1008 objective = self.verify_objective(objective)
1009 if isinstance(objective, str):
1010 objective = resolve_objective(objective)
1011 # Filter objective
1012 subdf = self.xs(objective.name, level=0, drop_level=True)
1013 # Filter solver
1014 subdf = subdf.xs(solver, axis=1, drop_level=True)
1015 # Drop the seed, then drop meta level as it is no longer needed
1016 subdf = subdf.drop(
1017 PerformanceDataFrame.column_seed,
1018 axis=1,
1019 level=PerformanceDataFrame.column_meta,
1020 )
1021 subdf = subdf.droplevel(PerformanceDataFrame.column_meta, axis=1)
1022 # Ensure the objective is numeric
1023 subdf = subdf.astype(float)
1025 if instance_pairs: # Filter instances
1026 pair_idx = pd.MultiIndex.from_tuples(instance_pairs)
1027 mask = subdf.index.droplevel(PerformanceDataFrame.index_run).isin(pair_idx)
1028 subdf = subdf[mask]
1029 if configuration: # Filter configuration
1030 if not isinstance(configuration, list):
1031 configuration = [configuration]
1032 subdf = subdf.filter(configuration, axis=1)
1033 # Aggregate the runs (by Instance level name)
1034 subdf = subdf.groupby(
1035 [
1036 PerformanceDataFrame.index_instance_set,
1037 PerformanceDataFrame.index_instance,
1038 ]
1039 ).agg(func=objective.run_aggregator.__name__)
1040 # Aggregate the instances
1041 sub_series = subdf.agg(func=objective.instance_aggregator.__name__)
1042 sub_series = sub_series.dropna()
1043 if sub_series.empty: # If all values are NaN, raise an error
1044 raise ValueError(
1045 f"No valid performance measurements for solver '{solver}' (Configuration: '{configuration}') "
1046 f"and objective '{objective.name}'."
1047 )
1048 # Select the best configuration
1049 best_conf = sub_series.idxmin() if objective.minimise else sub_series.idxmax()
1050 if per_instance: # Return a list of instance results
1051 return best_conf, subdf[best_conf].to_list()
1052 return best_conf, sub_series[best_conf]
1054 def best_configuration(
1055 self: PerformanceDataFrame,
1056 solver: str,
1057 objective: SparkleObjective = None,
1058 instance_pairs: list[tuple[str, str]] = None,
1059 ) -> tuple[str, float]:
1060 """Return the best configuration for the given objective over the instances.
1062 Args:
1063 solver: The solver for which we determine the best configuration
1064 objective: The objective for which we calculate the best configuration
1065 instance_pairs: The (set_name, instance_name) pairs to evaluate
1067 Returns:
1068 The best configuration id and its aggregated performance.
1069 """
1070 return self.configuration_performance(solver, None, objective, instance_pairs)
1072 def best_instance_performance(
1073 self: PerformanceDataFrame,
1074 objective: str | SparkleObjective = None,
1075 instance_pairs: list[tuple[str, str]] = None,
1076 run_id: int = None,
1077 exclude_solvers: list[(str, str)] = None,
1078 ) -> pd.Series:
1079 """Return the best performance for each instance in the portfolio.
1081 Args:
1082 objective: The objective for which we calculate the best performance
1083 instance_pairs: The (set_name, instance_name) pairs to evaluate
1084 run_id: The run for which we calculate the best performance. If None,
1085 we consider all runs.
1086 exclude_solvers: List of (solver, config_id) to exclude in the calculation.
1088 Returns:
1089 The best performance for each instance in the portfolio.
1090 """
1091 objective = self.verify_objective(objective)
1092 if isinstance(objective, str):
1093 objective = resolve_objective(objective)
1094 subdf = self.drop( # Drop Seed, not needed
1095 [PerformanceDataFrame.column_seed],
1096 axis=1,
1097 level=PerformanceDataFrame.column_meta,
1098 )
1099 subdf = subdf.xs(
1100 objective.name, level=PerformanceDataFrame.index_objective
1101 ) # Drop objective -> (InstanceSet, Instance, Run)
1102 if exclude_solvers is not None:
1103 subdf = subdf.drop(exclude_solvers, axis=1)
1104 if instance_pairs is not None:
1105 # subdf is (InstanceSet, Instance, Run) here. A plain .loc with 2-tuples would
1106 # misalign against the 3-level index. Mask on the (InstanceSet, Instance) pair
1107 # with Run dropped, mirroring configuration_performance's filter. An empty pair
1108 # list selects nothing (from_tuples([]) cannot infer levels, so short-circuit).
1109 if len(instance_pairs) == 0:
1110 subdf = subdf.iloc[:0]
1111 else:
1112 pair_idx = pd.MultiIndex.from_tuples(instance_pairs)
1113 mask = subdf.index.droplevel(PerformanceDataFrame.index_run).isin(
1114 pair_idx
1115 )
1116 subdf = subdf[mask]
1117 if run_id is not None:
1118 run_id = self.verify_run_id(run_id)
1119 subdf = subdf.xs(run_id, level=PerformanceDataFrame.index_run)
1120 else:
1121 # Drop the run level
1122 subdf = subdf.droplevel(PerformanceDataFrame.index_run)
1123 # Ensure the objective is numeric
1124 subdf = subdf.astype(float)
1125 series = subdf.min(axis=1) if objective.minimise else subdf.max(axis=1)
1126 # Ensure we always return the best for each run
1127 series = series.sort_values(ascending=objective.minimise)
1128 return series.groupby(series.index).first().astype(float)
1130 def best_performance(
1131 self: PerformanceDataFrame,
1132 exclude_solvers: list[(str, str)] = [],
1133 instance_pairs: list[tuple[str, str]] = None,
1134 objective: str | SparkleObjective = None,
1135 ) -> float:
1136 """Return the overall best performance of the portfolio.
1138 Args:
1139 exclude_solvers: List of (solver, config_id) to exclude in the calculation.
1140 Defaults to none.
1141 instance_pairs: The (set_name, instance_name) pairs to evaluate.
1142 If None, use all instances.
1143 objective: The objective for which we calculate the best performance
1145 Returns:
1146 The aggregated best performance of the portfolio over all instances.
1147 """
1148 objective = self.verify_objective(objective)
1149 if isinstance(objective, str):
1150 objective = resolve_objective(objective)
1151 instance_best = self.best_instance_performance(
1152 objective, instance_pairs=instance_pairs, exclude_solvers=exclude_solvers
1153 ).to_numpy(dtype=float)
1154 return objective.instance_aggregator(instance_best)
1156 def schedule_performance(
1157 self: PerformanceDataFrame,
1158 schedule: dict[tuple[str, str] : dict[str : (str, str, int)]],
1159 target_solver: str | tuple[str, str] = None,
1160 objective: str | SparkleObjective = None,
1161 ) -> float:
1162 """Return the performance of a selection schedule on the portfolio.
1164 Args:
1165 schedule: Compute the best performance according to a selection schedule.
1166 A schedule is a dictionary of (set_name, instance_name) pairs, with a
1167 schedule per instance, consisting of a triple of solver, config_id and
1168 maximum runtime.
1169 target_solver: If not None, store the found values in this solver of the DF.
1170 objective: The objective for which we calculate the best performance
1172 Returns:
1173 The performance of the schedule over the instances in the dictionary.
1174 """
1175 objective = self.verify_objective(objective)
1176 if isinstance(objective, str):
1177 objective = resolve_objective(objective)
1178 select = min if objective.minimise else max
1179 performances = [0.0] * len(schedule.keys())
1180 if not isinstance(target_solver, tuple):
1181 target_conf = PerformanceDataFrame.default_configuration
1182 else:
1183 target_solver, target_conf = target_solver
1184 if target_solver and target_solver not in self.solvers:
1185 self.add_solver(target_solver)
1186 for ix, instance_pair in enumerate(schedule.keys()):
1187 for iy, (solver, config, max_runtime) in enumerate(schedule[instance_pair]):
1188 performance = float(
1189 self.get_value(solver, instance_pair, config, objective.name)
1190 )
1191 if max_runtime is not None: # We are dealing with runtime
1192 performances[ix] += performance
1193 if performance < max_runtime:
1194 break # Solver finished in time
1195 else: # Quality, we take the best found performance
1196 if iy == 0: # First solver, set initial value
1197 performances[ix] = performance
1198 continue
1199 performances[ix] = select(performances[ix], performance)
1200 if target_solver is not None:
1201 self.set_value(
1202 performances[ix],
1203 target_solver,
1204 instance_pair,
1205 target_conf,
1206 objective.name,
1207 )
1208 return performances
1210 def marginal_contribution(
1211 self: PerformanceDataFrame,
1212 objective: str | SparkleObjective = None,
1213 instance_pairs: list[tuple[str, str]] = None,
1214 sort: bool = False,
1215 ) -> list[float]:
1216 """Return the marginal contribution of the solver configuration on the instances.
1218 Args:
1219 objective: The objective for which we calculate the marginal contribution.
1220 instance_pairs: The (set_name, instance_name) pairs to evaluate
1221 sort: Whether to sort the results afterwards
1222 Returns:
1223 The marginal contribution of each solver (configuration) as:
1224 [(solver, config_id, marginal_contribution, portfolio_best_performance_without_solver)]
1225 """
1226 output = []
1227 objective = self.verify_objective(objective)
1228 if isinstance(objective, str):
1229 objective = resolve_objective(objective)
1230 best_performance = self.best_performance(
1231 objective=objective, instance_pairs=instance_pairs
1232 )
1233 for solver in self.solvers:
1234 for config_id in self.get_configurations(solver):
1235 # By calculating the best performance excluding this Solver,
1236 # we can determine its relative impact on the portfolio.
1237 missing_solver_config_best = self.best_performance(
1238 exclude_solvers=[(solver, config_id)],
1239 instance_pairs=instance_pairs,
1240 objective=objective,
1241 )
1242 # Now we need to see how much the portfolio's best performance
1243 # decreases without this solver.
1244 marginal_contribution = missing_solver_config_best / best_performance
1245 if missing_solver_config_best == best_performance:
1246 # No change, no contribution
1247 marginal_contribution = 0.0
1248 output.append(
1249 (
1250 solver,
1251 config_id,
1252 marginal_contribution,
1253 missing_solver_config_best,
1254 )
1255 )
1256 if sort:
1257 output.sort(key=lambda x: x[2], reverse=objective.minimise)
1258 return output
1260 def get_solver_ranking(
1261 self: PerformanceDataFrame,
1262 objective: str | SparkleObjective = None,
1263 instance_pairs: list[tuple[str, str]] = None,
1264 ) -> list[tuple[str, dict, float]]:
1265 """Return a list with solvers ranked by average performance."""
1266 objective = self.verify_objective(objective)
1267 if isinstance(objective, str):
1268 objective = resolve_objective(objective)
1269 # Drop Seed
1270 sub_df = self.drop(
1271 [PerformanceDataFrame.column_seed],
1272 axis=1,
1273 level=PerformanceDataFrame.column_meta,
1274 )
1275 # Reduce objective (4-level index -> 3-level: InstanceSet, Instance, Run)
1276 sub_df: pd.DataFrame = sub_df.loc(axis=0)[objective.name, :, :, :]
1277 # Drop Objective, Meta multi index
1278 sub_df = sub_df.droplevel(PerformanceDataFrame.index_objective).droplevel(
1279 PerformanceDataFrame.column_meta, axis=1
1280 )
1281 if instance_pairs is not None: # Select instances
1282 # sub_df is (InstanceSet, Instance, Run) mask on the (InstanceSet, Instance)
1283 # pair with Run dropped rather than .loc with 2-tuples (which misaligns).
1284 if len(instance_pairs) == 0:
1285 sub_df = sub_df.iloc[:0]
1286 else:
1287 pair_idx = pd.MultiIndex.from_tuples(instance_pairs)
1288 mask = sub_df.index.droplevel(PerformanceDataFrame.index_run).isin(
1289 pair_idx
1290 )
1291 sub_df = sub_df[mask]
1292 # Ensure data is numeric
1293 sub_df = sub_df.astype(float)
1294 # Aggregate runs (by Instance level name collapses InstanceSet and Instance into Instance)
1295 sub_df = sub_df.groupby(PerformanceDataFrame.index_instance).agg(
1296 func=objective.run_aggregator.__name__
1297 )
1298 # Aggregate instances
1299 sub_series = sub_df.aggregate(func=objective.instance_aggregator.__name__)
1300 # Sort by objective
1301 sub_series.sort_values(ascending=objective.minimise, inplace=True)
1302 return [(index[0], index[1], sub_series[index]) for index in sub_series.index]
1304 def save_csv(self: PerformanceDataFrame, csv_filepath: Path = None) -> None:
1305 """Write a CSV to the given path.
1307 Args:
1308 csv_filepath: String path to the csv file. Defaults to self.csv_filepath.
1309 """
1310 csv_filepath = self.csv_filepath if csv_filepath is None else csv_filepath
1311 self.to_csv(csv_filepath)
1312 # Append the configurations
1313 with csv_filepath.open("a") as fout:
1314 fout.write("\n$Solver,configuration_id,Configuration\n")
1315 for solver in self.solvers:
1316 for config_id in self.attrs[solver]:
1317 configuration = self.attrs[solver][config_id]
1318 fout.write(f"${solver},{config_id},{str(configuration)}\n")
1320 def clone(
1321 self: PerformanceDataFrame, csv_filepath: Path = None
1322 ) -> PerformanceDataFrame:
1323 """Create a copy of this object.
1325 Args:
1326 csv_filepath: The new filepath to use for saving the object to.
1327 If None, will not be saved.
1328 Warning: If the original path is used, it could lead to dataloss!
1329 """
1330 pd_copy = PerformanceDataFrame(
1331 csv_filepath=csv_filepath,
1332 solvers=self.solvers,
1333 configurations=self.configurations,
1334 objectives=self.objectives,
1335 instance_pairs=self.instance_pairs,
1336 n_runs=self.num_runs,
1337 )
1338 # Copy values
1339 for column_index in self.columns:
1340 for index in self.index:
1341 pd_copy.at[index, column_index] = self.loc[index, column_index]
1342 # Ensure everything is sorted?
1343 return pd_copy
1345 def clean_csv(self: PerformanceDataFrame) -> None:
1346 """Set all values in Performance Data to None."""
1347 self[:] = PerformanceDataFrame.missing_value
1348 self.save_csv()