Merge unstable into main: solver config interface and test coverage

README.md

@@ -178,7 +178,7 @@ config = TemoaConfig(
     time_sequencing='seasonal_timeslices',
     input_database='tutorial_database.sqlite',
     output_database='tutorial_database.sqlite',
-    solver_name='appsi_highs',
+    solver='appsi_highs',
     output_path=output_path,
     silent=False,
 )
@@ -238,7 +238,7 @@ scenario = "tutorial"
 scenario_mode = "perfect_foresight"
 input_database = "tutorial_database.sqlite"
 output_database = "tutorial_database.sqlite"
-solver_name = "appsi_highs"
+solver = "appsi_highs"
 ```
 
 ### Configuration Options

requirements-dev.txt

@@ -1148,9 +1148,9 @@ pillow==12.3.0 \
     --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \
     --hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7
     # via matplotlib
-pint==0.25.3 \
-    --hash=sha256:27eb25143bd5de9fcc4d5a4b484f16faf6b4615aa93ece6b3373a8c1a3c1b97d \
-    --hash=sha256:f8f5df6cf65314d74da1ade1bf96f8e3e4d0c41b51577ac53c49e7d44ca5acee
+pint==0.26.1 \
+    --hash=sha256:1bbde36eae57a5a289cd05081c6405618a5899814940752064ce351cd0204f71 \
+    --hash=sha256:e982b129415c09c63308f314ae44697d83e5c96f253bcc0d5b833fc3329eb6d4
     # via temoa
 platformdirs==4.11.7 \
     --hash=sha256:4f41487eeeeeb07f3a6625e61d9bc0ae6809f92d3386dbd74392fbb76108104d \

requirements.txt

@@ -557,9 +557,9 @@ pillow==12.3.0 \
     --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \
     --hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7
     # via matplotlib
-pint==0.25.3 \
-    --hash=sha256:27eb25143bd5de9fcc4d5a4b484f16faf6b4615aa93ece6b3373a8c1a3c1b97d \
-    --hash=sha256:f8f5df6cf65314d74da1ade1bf96f8e3e4d0c41b51577ac53c49e7d44ca5acee
+pint==0.26.1 \
+    --hash=sha256:1bbde36eae57a5a289cd05081c6405618a5899814940752064ce351cd0204f71 \
+    --hash=sha256:e982b129415c09c63308f314ae44697d83e5c96f253bcc0d5b833fc3329eb6d4
     # via temoa
 platformdirs==4.11.7 \
     --hash=sha256:4f41487eeeeeb07f3a6625e61d9bc0ae6809f92d3386dbd74392fbb76108104d \

temoa/_internal/run_actions.py

@@ -3,12 +3,13 @@
 """
 
 import sqlite3
-from collections.abc import Generator, Iterable
+from collections.abc import Generator, Iterable, Mapping
 from contextlib import contextmanager
 from logging import getLogger
 from pathlib import Path
 from sys import version_info
 from time import perf_counter
+from typing import Any
 
 from pyomo.environ import (
     Constraint,
@@ -25,6 +26,7 @@
 from temoa._internal.table_writer import TableWriter
 from temoa.core.config import TemoaConfig
 from temoa.core.model import TemoaModel
+from temoa.core.solver_spec import DEFAULT_SOLVER_OPTIONS
 from temoa.data_processing.db_to_excel import make_excel
 
 logger = getLogger(__name__)
@@ -174,13 +176,16 @@ def solve_instance(
     solver_name: str,
     silent: bool = False,
     solver_suffixes: Iterable[str] | None = None,
+    solver_options: Mapping[str, Any] | None = None,
 ) -> tuple[TemoaModel, SolverResults]:
     """
     Solve the instance and return a loaded instance
     :param solver_suffixes: iterable of string names for suffixes.  See pyomo dox.  right now, only
     'duals' is supported in the Temoa Framework.  Some solvers may not support duals.
     :param silent: Run silently
     :param solver_name: The name of the solver to request from the SolverFactory
+    :param solver_options: options to set on the solver (see resolve_solver_options).  If None,
+    the Temoa defaults for the solver are used
     :param instance: the instance to solve
     :return: loaded instance
     """
@@ -202,28 +207,11 @@ def solve_instance(
     if solver_name == 'neos':
         raise NotImplementedError('Neos based solve is not currently supported')
 
-    # Solver Configuration
-    if solver_name == 'cbc':
-        pass
-
-    elif solver_name == 'cplex':
-        # Note: these parameter values match mip-dev / PyPSA
-        # (see: https://pypsa-eur.readthedocs.io/en/latest/configuration.html)
-        optimizer.options['lpmethod'] = 4  # barrier
-        optimizer.options['solutiontype'] = 2  # non basic solution, ie no crossover
-        optimizer.options['barrier convergetol'] = 1.0e-3
-        optimizer.options['feasopt tolerance'] = 1.0e-4
-
-    elif solver_name == 'gurobi':
-        # Note: these parameter values match mip-dev / PyPSA (see: https://pypsa-eur.readthedocs.io/en/latest/configuration.html)
-        optimizer.options['Method'] = 2  # barrier
-        optimizer.options['Crossover'] = 0  # non basic solution, ie no crossover
-        optimizer.options['BarConvTol'] = 1.0e-3
-        optimizer.options['FeasibilityTol'] = 1.0e-4
-        optimizer.options['BarOrder'] = -1  # auto ordering; 2-4x faster than AMD on large models
-
-    elif solver_name == 'appsi_highs':
-        pass
+    # Solver Configuration (defaults live in temoa.core.solver_spec.DEFAULT_SOLVER_OPTIONS)
+    if solver_options is None:
+        solver_options = DEFAULT_SOLVER_OPTIONS.get(solver_name, {})
+    for option, option_value in solver_options.items():
+        optimizer.options[option] = option_value
 
     # Suffix Handling
     solver_suffixes_list: list[str] = []

temoa/_internal/temoa_sequencer.py

@@ -27,6 +27,7 @@
 from temoa.core.config import TemoaConfig
 from temoa.core.model import TemoaModel
 from temoa.core.modes import TemoaMode
+from temoa.core.solver_spec import resolve_solver_options
 from temoa.data_io.hybrid_loader import HybridLoader
 from temoa.extensions.method_of_morris.morris_sequencer import MorrisSequencer
 from temoa.extensions.modeling_to_generate_alternatives.mga_sequencer import MgaSequencer
@@ -254,6 +255,7 @@ def _run_perfect_foresight(self) -> None:
                 self.config.solver_name,
                 silent=self.config.silent,
                 solver_suffixes=suffixes,
+                solver_options=resolve_solver_options(self.config.solver),
             )
             good_solve, msg = check_solve_status(self.pf_results)
             if not good_solve:

temoa/core/config.py

@@ -1,10 +1,14 @@
 import shutil
 import sys
 import tomllib
+import warnings
+from collections.abc import Mapping
 from logging import getLogger
 from pathlib import Path
+from typing import Any
 
 from temoa.core.modes import TemoaMode
+from temoa.core.solver_spec import SolverSpec, redact_solver_options
 from temoa.extensions.framework import normalize_extension_ids, resolve_extension_specs
 
 logger = getLogger(__name__)
@@ -42,7 +46,7 @@ def __init__(
         input_database: Path,
         output_database: Path,
         output_path: Path,
-        solver_name: str,
+        solver_name: str | None = None,
         neos: bool = False,
         save_excel: bool = False,
         save_duals: bool = False,
@@ -73,6 +77,7 @@ def __init__(
         output_threshold_cost: float | None = None,
         sqlite: dict[str, object] | None = None,
         extensions: list[str] | tuple[str, ...] | None = None,
+        solver: str | Mapping[str, Any] | SolverSpec | None = None,
     ):
         if '-' in scenario:
             raise ValueError(
@@ -129,7 +134,20 @@ def __init__(
         self.neos = neos
         if self.neos:
             raise NotImplementedError('Neos is currently not supported.')
-        self.solver_name = solver_name
+
+        # Validate solver input
+        if solver_name is not None:
+            if solver is not None:
+                raise ValueError("Specify either 'solver' or 'solver_name', not both")
+            warnings.warn(
+                "The 'solver_name' argument is deprecated, use 'solver' instead",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            solver = solver_name
+        if solver is None:
+            raise SolverNotAvailableError('No solver specified in the configuration.')
+        self.solver = SolverSpec.parse(solver)
 
         self.save_excel = save_excel
         self.save_duals = save_duals
@@ -230,6 +248,17 @@ def __init__(
                 if not self.silent:
                     sys.stderr.write('Warning: ' + msg)
 
+    @property
+    def solver_name(self) -> str:
+        """The name of the selected solver (shorthand for self.solver.name)"""
+        return self.solver.name
+
+    @solver_name.setter
+    def solver_name(self, value: str) -> None:
+        # retained for backward compatibility.  Changing solvers drops any configured options,
+        # as they are specific to the previous solver
+        self.solver = SolverSpec.parse(value)
+
     @staticmethod
     def _check_solver_availability(solver_name: str) -> tuple[bool, str | None]:
         """
@@ -282,27 +311,41 @@ def build_config(config_file: Path, output_path: Path, silent: bool = False) ->
             data = tomllib.load(f)
 
         if 'solver_name' in data:
-            is_available, location = TemoaConfig._check_solver_availability(data['solver_name'])
-            if not is_available:
-                error_message = (
-                    f"The specified solver '{data['solver_name']}' was not found.\n"
-                    'Please ensure the solver is installed and accessible.\n'
+            if 'solver' in data:
+                raise ValueError(
+                    "Config specifies both 'solver' and 'solver_name'.  Use only 'solver' "
+                    "('solver_name' is deprecated)."
                 )
-                if data['solver_name'].lower() in SOLVER_DOC_LINKS:
-                    link = SOLVER_DOC_LINKS[data['solver_name'].lower()]
-                    error_message += f'For installation instructions, refer to: {link}\n'
-                else:
-                    error_message += (
-                        "Refer to the solver's official documentation for "
-                        'installation instructions.'
-                    )
-                raise SolverNotAvailableError(error_message)
+            logger.warning(
+                "The 'solver_name' config key is deprecated and will be removed in a future "
+                'release.  Replace it with: solver = "%s"',
+                data['solver_name'],
+            )
+            data['solver'] = data.pop('solver_name')
+
+        if 'solver' not in data:
+            raise SolverNotAvailableError('No solver name specified in the configuration.')
+        solver = SolverSpec.parse(data['solver'])
+        data['solver'] = solver
+
+        is_available, location = TemoaConfig._check_solver_availability(solver.name)
+        if not is_available:
+            error_message = (
+                f"The specified solver '{solver.name}' was not found.\n"
+                'Please ensure the solver is installed and accessible.\n'
+            )
+            if solver.name.lower() in SOLVER_DOC_LINKS:
+                link = SOLVER_DOC_LINKS[solver.name.lower()]
+                error_message += f'For installation instructions, refer to: {link}\n'
             else:
-                logger.info('Using solver: %s (%s)', data['solver_name'], location)
+                error_message += (
+                    "Refer to the solver's official documentation for installation instructions."
+                )
+            raise SolverNotAvailableError(error_message)
         else:
-            raise SolverNotAvailableError('No solver name specified in the configuration.')
+            logger.info('Using solver: %s (%s)', solver.name, location)
 
-        if data.get('solver_name') == 'appsi_highs' and data.get('save_duals', False):
+        if solver.name == 'appsi_highs' and data.get('save_duals', False):
             raise ValueError(
                 'save_duals is not supported with appsi_highs (it does not expose duals via the '
                 'APPSI interface). Disable save_duals or choose a different solver.'
@@ -354,6 +397,9 @@ def __repr__(self) -> str:
 
         msg += spacer
         msg += '{:>{}s}: {}\n'.format('Selected solver', width, self.solver_name)
+        msg += '{:>{}s}: {}\n'.format(
+            'Solver options', width, redact_solver_options(self.solver.options)
+        )
         msg += '{:>{}s}: {}\n'.format('NEOS status', width, self.neos)
 
         msg += spacer

temoa/core/solver_spec.py

@@ -0,0 +1,107 @@
+"""
+Solver selection and option resolution.
+
+The config accepts ``solver`` as either a plain solver name or a table with a ``name`` and a
+passthrough ``options`` table.  Options that reach the solver are layered as:
+
+    DEFAULT_SOLVER_OPTIONS  <  [solver.options]  <  extension-specific options (MGA, MC, ...)
+
+Any solver name known to pyomo's SolverFactory is accepted.  Solvers without an entry in
+DEFAULT_SOLVER_OPTIONS simply get no Temoa defaults.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+# Note: these parameter values match mip-dev / PyPSA
+# (see: https://pypsa-eur.readthedocs.io/en/latest/configuration.html)
+DEFAULT_SOLVER_OPTIONS: dict[str, dict[str, Any]] = {
+    'cplex': {
+        'lpmethod': 4,  # barrier
+        'solutiontype': 2,  # non basic solution, ie no crossover
+        'barrier convergetol': 1.0e-3,
+        'feasopt tolerance': 1.0e-4,
+    },
+    'gurobi': {
+        'Method': 2,  # barrier
+        'Crossover': 0,  # non basic solution, ie no crossover
+        'BarConvTol': 1.0e-3,
+        'FeasibilityTol': 1.0e-4,
+        'BarOrder': -1,  # auto ordering; 2-4x faster than AMD on large models
+    },
+}
+
+
+@dataclass(frozen=True, slots=True)
+class SolverSpec:
+    """A solver name plus the user-supplied options from the config (defaults excluded)."""
+
+    name: str
+    options: Mapping[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def parse(cls, raw: str | Mapping[str, Any] | SolverSpec) -> SolverSpec:
+        """
+        Build a SolverSpec from a solver name, a {'name': ..., 'options': {...}} mapping, or an
+        existing SolverSpec
+        """
+        if isinstance(raw, SolverSpec):
+            return raw
+        if isinstance(raw, str):
+            if not raw:
+                raise ValueError('Solver name must not be empty')
+            return cls(raw)
+        if isinstance(raw, Mapping):
+            unknown = set(raw) - {'name', 'options'}
+            if unknown:
+                raise ValueError(
+                    f'Unrecognized key(s) in solver table: {sorted(unknown)}.  Expected "name" '
+                    'and (optionally) "options".  Solver parameters belong under [solver.options]'
+                )
+            name = raw.get('name')
+            if not isinstance(name, str) or not name:
+                raise ValueError('The solver table requires a non-empty "name" entry')
+            options = raw.get('options', {})
+            if not isinstance(options, Mapping):
+                raise ValueError('The "options" entry of the solver table must be a table/dict')
+            return cls(name, dict(options))
+        raise TypeError(f'solver must be a str or a table/dict, got: {type(raw).__name__}')
+
+
+# substrings (lowercase) of option names whose values are credentials, e.g. gurobi's WLSSecret,
+# CloudSecretKey, CSAPIAccessID, ServerPassword, LicenseID
+_SENSITIVE_OPTION_MARKERS = ('secret', 'password', 'accessid', 'licenseid', 'key', 'token')
+
+
+def redact_solver_options(options: Mapping[str, Any]) -> dict[str, Any]:
+    """
+    Return a copy of the options that is safe to log or print, with credential values masked
+    """
+    return {
+        option: '***'
+        if any(marker in option.lower() for marker in _SENSITIVE_OPTION_MARKERS)
+        else option_value
+        for option, option_value in options.items()
+    }
+
+
+def resolve_solver_options(
+    spec: SolverSpec,
+    extension_options: Mapping[str, Any] | None = None,
+    *,
+    include_defaults: bool = True,
+) -> dict[str, Any]:
+    """
+    Merge the option layers that are passed to the solver
+    :param spec: the solver spec from the config
+    :param extension_options: options from an extension's own source (MGA/MC/Morris toml,
+    stochastic config), which take precedence over everything else
+    :param include_defaults: include DEFAULT_SOLVER_OPTIONS as the bottom layer.  Solve paths that
+    historically ran without Temoa defaults (MGA base solve, stochastic) turn this off.
+    :return: a new dict of solver options
+    """
+    defaults = DEFAULT_SOLVER_OPTIONS.get(spec.name, {}) if include_defaults else {}
+    return {**defaults, **spec.options, **(extension_options or {})}