Merge remote-tracking branch 'origin/main' into feature/movingWindow

Author: KaStephan

CaseStudy.py

@@ -36,7 +36,6 @@ def check_LEGOExcel_version(xls: pd.ExcelFile, sheet_name: str, version_specifie
                 f"Excel file '{excel_file_path}' (sheet '{sheet_name}') does not have the correct version specifier. "
                 f"Expected '{version_specifier}' but got '{version_cell}'.")
             printer.error(f"Trying to work with it any way, but this can have unintended consequences!")
-    pass
 
 
 def __read_non_pivoted_file(excel_file_path: str, version_specifier: str, indices: list[str], has_excl_column: bool,
@@ -348,6 +347,11 @@ def get_Power_Network(excel_file_path: str, keep_excluded_entries: bool = False,
     """
     dPower_Network = __read_non_pivoted_file(excel_file_path, "v0.1.2", ["i", "j", "c"], True, keep_excluded_entries, fail_on_wrong_version)
 
+    # Check that all values in column pEnableInvest are either 0 or 1
+    if not dPower_Network['pEnableInvest'].isin([0, 1]).all():
+        invalid_values = dPower_Network.loc[~dPower_Network['pEnableInvest'].isin([0, 1]), 'pEnableInvest']
+        raise ValueError(f"dPower_Network: Found invalid values in 'pEnableInvest' column. Only 0 and 1 are allowed, but found: {invalid_values}")
+
     return dPower_Network
 
 
@@ -519,8 +523,10 @@ def compare_Excels(source_path: str, target_path: str, dont_check_formatting: bo
                     for k, v in source_cell.font.__dict__.items():
                         if k == "color" and v is not None:
                             for k2, v2 in v.__dict__.items():
-                                if v2 != getattr(target_cell.font.color, k2):
-                                    printer.error(f"Mismatch in font color at {sheet}/{source_cell.coordinate}: {v2} != {getattr(target_cell.font.color, k2)}")
+                                if ((v2 is None and target_cell.font.color is not None) or
+                                        (v2 is not None and target_cell.font.color is None) or
+                                        (v2 != getattr(target_cell.font.color, k2))):
+                                    printer.error(f"Mismatch in font color at {sheet}/{source_cell.coordinate}: {v2} != {getattr(target_cell.font.color, k2) if target_cell.font.color is not None else None}")
                                     equal = False
                         elif getattr(target_cell.font, k) != v:
                             printer.error(f"Mismatch in font property '{k}' at {sheet}/{source_cell.coordinate}: {getattr(target_cell.font, k)} != {v}")
@@ -530,8 +536,10 @@ def compare_Excels(source_path: str, target_path: str, dont_check_formatting: bo
                     for k, v in source_cell.fill.__dict__.items():
                         if k == "color" and v is not None:
                             for k2, v2 in v.__dict__.items():
-                                if v2 != getattr(target_cell.fill.color, k2):
-                                    printer.error(f"Mismatch in fill color at {sheet}/{source_cell.coordinate}: {v2} != {getattr(target_cell.fill.color, k2)}")
+                                if ((v2 is None and target_cell.fill.color is not None) or
+                                        (v2 is not None and target_cell.fill.color is None) or
+                                        (v2 != getattr(target_cell.fill.color, k2))):
+                                    printer.error(f"Mismatch in fill color at {sheet}/{source_cell.coordinate}: {v2} != {getattr(target_cell.fill.color, k2) if target_cell.fill.color is not None else None}")
                                     equal = False
                         elif getattr(target_cell.fill, k) != v:
                             printer.error(f"Mismatch in fill property '{k}' at {sheet}/{source_cell.coordinate}: {getattr(target_cell.fill, k)} != {v}")

ExcelReader.py

@@ -1,8 +1,11 @@
+from __future__ import annotations
+
 import os
 import time
 import xml.etree.ElementTree as ET
 from copy import copy, deepcopy
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import numpy as np
 import openpyxl
@@ -12,7 +15,9 @@
 
 import ExcelReader
 import TableDefinition
-from CaseStudy import CaseStudy
+
+if TYPE_CHECKING:
+    from CaseStudy import CaseStudy
 from TableDefinition import CellStyle, Alignment, Font, Color, Text, Column, NumberFormat, TableDefinition
 from printer import Printer
 
@@ -262,6 +267,8 @@ def write_caseStudy(self, cs: CaseStudy, folder_path: str | Path) -> None:
             self.write_Power_VRES(cs.dPower_VRES, folder_path)
         if hasattr(cs, "dPower_VRESProfiles"):
             self.write_Power_VRESProfiles(cs.dPower_VRESProfiles, folder_path)
+        if hasattr(cs, "dPower_ImportExport") and cs.dPower_ImportExport is not None:
+            self.write_Power_ImportExport(cs.dPower_ImportExport, folder_path)
         self.write_Power_WeightsK(cs.dPower_WeightsK, folder_path)
         self.write_Power_WeightsRP(cs.dPower_WeightsRP, folder_path)
 
@@ -597,3 +604,6 @@ def model_to_excel(model: pyomo.core.Model, target_path: str) -> None:
         for excel_definition_id, file_path in files_different:
             printer.error(f"'{excel_definition_id}' (file: '{file_path}')")
         exit(1)
+    else:
+        printer.success("All Excel files were equal")
+        exit(0)

ExcelWriter.py

@@ -289,10 +289,247 @@ def model_to_sqlite(model: pyo.base.Model, filename: str, use_moving_window: boo
                     df = pd.DataFrame([pyo.value(o)], columns=['values'])
                 case pyomo.core.base.constraint.ConstraintList | pyomo.core.base.constraint.IndexedConstraint | pyomo.core.base.expression.IndexedExpression:  # Those will not be saved on purpose
                     continue
+                case pyomo.core.base.suffix.Suffix:
+                    if str(o) in ["_relaxed_integer_vars", "dual"]:
+                        continue  # Not saved on purpose
+                    else:
+                        printer.warning(f"Pyomo-Type {type(o)} not implemented, {o.name} will not be saved to SQLite")
+                        continue
                 case _:
                     printer.warning(f"Pyomo-Type {type(o)} not implemented, {o.name} will not be saved to SQLite")
                     continue
-            df.to_sql(o.name, cnx, if_exists='replace')
+        df.to_sql(o.name, cnx, if_exists='replace')
+        cnx.commit()
+    cnx.close()
+
+    # Automatically add objective decomposition and dual values
+    add_objective_decomposition_to_sqlite(filename, model)
+    add_dual_values_to_sqlite(filename, model)
+    pass
+
+
+def add_solver_statistics_to_sqlite(filename: str, results, work_units=None) -> None:
+    """
+    Add solver statistics (like Gurobi work-units) to an existing SQLite database.
+    :param filename: Path to the SQLite database file
+    :param results: Pyomo solver results object
+    :param work_units: Optional work units value (from Gurobi solver)
+    :return: None
+    """
+    cnx = sqlite3.connect(filename)
+
+    # Extract solver statistics
+    stats = {}
+
+    try:
+        # Add work units if provided
+        if work_units is not None:
+            stats['work_units'] = float(work_units)
+
+        # Get basic solver info from solver[0]
+        if hasattr(results, 'solver') and len(results.solver) > 0:
+            solver_info = results.solver[0]
+
+            # Status and termination
+            if hasattr(solver_info, 'status'):
+                stats['solver_status'] = str(solver_info.status)
+            if hasattr(solver_info, 'termination_condition'):
+                stats['termination_condition'] = str(solver_info.termination_condition)
+            if hasattr(solver_info, 'time'):
+                try:
+                    time_val = solver_info.time
+                    if time_val is not None and str(type(time_val)) != "<class 'pyomo.opt.results.container.UndefinedData'>":
+                        stats['solver_time'] = float(time_val)
+                except Exception:
+                    pass
+
+        # Get problem statistics
+        if hasattr(results, 'problem'):
+            problem = results.problem
+            for attr in ['lower_bound', 'upper_bound', 'number_of_constraints',
+                         'number_of_variables', 'number_of_nonzeros']:
+                if hasattr(problem, attr):
+                    value = getattr(problem, attr)
+                    if value is not None:
+                        stats[attr] = float(value) if isinstance(value, (int, float)) else str(value)
+
+        # Create a DataFrame with solver statistics
+        if stats:
+            df = pd.DataFrame([stats])
+            df.to_sql('solver_statistics', cnx, if_exists='replace', index=False)
             cnx.commit()
+            work_units_str = f"{stats['work_units']:.2f}" if 'work_units' in stats else 'N/A'
+            printer.information(f"Added solver statistics to SQLite database (work_units: {work_units_str})")
+        else:
+            printer.warning("No solver statistics found in results object")
+
+    except Exception as e:
+        printer.error(f"Failed to add solver statistics: {e}")
+        import traceback
+        traceback.print_exc()
+    finally:
+        cnx.close()
+
+
+def add_run_parameters_to_sqlite(filename: str, **parameters) -> None:
+    """
+    Add run parameters to an existing SQLite database.
+    Creates a table 'run_parameters' with parameter names and values.
+
+    :param filename: Path to the SQLite database file
+    :param parameters: Keyword arguments containing parameter names and values
+    :return: None
+
+    Example:
+        add_run_parameters_to_sqlite('model.sqlite',
+                                     zoi='R1',
+                                     dc_buffer=2,
+                                     tp_buffer=1,
+                                     scale_demand=1.3,
+                                     scale_pmax=1.0)
+    """
+    cnx = sqlite3.connect(filename)
+
+    try:
+        # Convert parameters to DataFrame
+        params = {}
+        for key, value in parameters.items():
+            # Convert None to string 'None' for storage
+            if value is None:
+                params[key] = 'None'
+            elif isinstance(value, (int, float)):
+                params[key] = float(value)
+            else:
+                params[key] = str(value)
+
+        if params:
+            df = pd.DataFrame([params])
+            df.to_sql('run_parameters', cnx, if_exists='replace', index=False)
+            cnx.commit()
+            printer.information(f"Added run parameters to SQLite database: {', '.join([f'{k}={v}' for k, v in params.items()])}")
+        else:
+            printer.warning("No run parameters provided")
+
+    except Exception as e:
+        printer.error(f"Failed to add run parameters: {e}")
+        import traceback
+        traceback.print_exc()
+    finally:
+        cnx.close()
+
+
+def add_objective_decomposition_to_sqlite(filename: str, model: pyo.ConcreteModel) -> None:
+    """
+    Add objective function decomposition to SQLite database.
+    This enables recalculation of ZOI objectives without the full model.
+
+    The objective is decomposed into:
+    - objective_constant: Single row with the constant term
+    - objective_terms: Variable names, indices, and their coefficients
+
+    :param filename: Path to the SQLite database file
+    :param model: Pyomo model with objective
+    :return: None
+    """
+    from pyomo.repn import generate_standard_repn
+
+    cnx = sqlite3.connect(filename)
+
+    try:
+        # Decompose objective into linear representation
+        repn = generate_standard_repn(model.objective.expr, quadratic=False)
+
+        # Store objective decomposition as separate tables
+        # 1. Constant term
+        df_constant = pd.DataFrame([{'constant': repn.constant if repn.constant else 0.0}])
+        df_constant.to_sql('objective_constant', cnx, if_exists='replace', index=False)
+
+        # 2. Variable names, indices, and coefficients
+        var_names = [var.parent_component().name for var in repn.linear_vars]
+        var_indices = [str(var.index()) for var in repn.linear_vars]
+        coefs = list(repn.linear_coefs)
+        df_terms = pd.DataFrame({
+            'var_name': var_names,
+            'var_index': var_indices,
+            'coefficient': coefs
+        })
+        df_terms.to_sql('objective_terms', cnx, if_exists='replace', index=False)
+
+        cnx.commit()
+        printer.information(f"Added objective decomposition to SQLite ({len(var_indices)} terms)")
+
+    except Exception as e:
+        printer.error(f"Failed to add objective decomposition: {e}")
+        import traceback
+        traceback.print_exc()
+    finally:
+        cnx.close()
+
+
+def add_dual_values_to_sqlite(filename: str, model: pyo.ConcreteModel) -> None:
+    """
+    Add dual values (shadow prices) from model constraints to SQLite database.
+
+    Dual values are stored in tables named 'dual_<constraint_name>' with:
+    - Index columns for the constraint
+    - 'dual_value' column containing the dual/shadow price
+
+    :param filename: Path to the SQLite database file
+    :param model: Solved Pyomo model with dual suffix
+    :return: None
+    """
+    cnx = sqlite3.connect(filename)
+
+    try:
+        if not hasattr(model, 'dual'):
+            printer.warning("Model does not have dual suffix - no dual values to save")
+            return
+
+        total_duals = 0
+        total_constraints = 0
+
+        # Iterate through all constraints and save their dual values
+        for constraint in model.component_objects(pyo.Constraint, active=True):
+            constraint_name = constraint.name
+            dual_data = []
+
+            # Get dual values for this constraint
+            for index in constraint:
+                try:
+                    dual_value = model.dual[constraint[index]]
+                    if dual_value is not None:
+                        # Store index and dual value
+                        if isinstance(index, tuple):
+                            # Multi-indexed constraint
+                            row_data = {str(i): val for i, val in enumerate(index)}
+                            row_data['dual_value'] = float(dual_value)
+                        else:
+                            # Single-indexed or scalar constraint
+                            row_data = {'0': index, 'dual_value': float(dual_value)}
+                        dual_data.append(row_data)
+                        total_duals += 1
+                except (KeyError, AttributeError):
+                    # Dual value not available for this constraint
+                    pass
+
+            total_constraints += 1
+
+            # Save to database if we have dual values for this constraint
+            if dual_data:
+                df = pd.DataFrame(dual_data)
+                table_name = f'dual_{constraint_name}'
+                df.to_sql(table_name, cnx, if_exists='replace', index=False)
+
+        cnx.commit()
+
+        if total_duals > 0:
+            printer.information(f"Added dual values to SQLite ({total_duals} duals from {total_constraints} constraints)")
+        else:
+            printer.warning(f"No dual values found in model (checked {total_constraints} constraints)")
+
+    except Exception as e:
+        printer.error(f"Failed to add dual values: {e}")
+        import traceback
+        traceback.print_exc()
+    finally:
         cnx.close()
-        pass