From 31ae3ff2bba4f5d814c9621c5e1201a9869cfe06 Mon Sep 17 00:00:00 2001 From: jasherma Date: Fri, 29 Nov 2024 21:32:29 -0500 Subject: [PATCH 01/36] Extend `InputDataStandardizer` to allow multiple component types --- pyomo/contrib/pyros/config.py | 81 ++++++++++++------------ pyomo/contrib/pyros/tests/test_config.py | 79 +++++++++++++++++++++-- 2 files changed, 112 insertions(+), 48 deletions(-) diff --git a/pyomo/contrib/pyros/config.py b/pyomo/contrib/pyros/config.py index 5abc61536cb..3f82c2e3ae8 100644 --- a/pyomo/contrib/pyros/config.py +++ b/pyomo/contrib/pyros/config.py @@ -2,10 +2,8 @@ Interfaces for managing PyROS solver options. """ -from collections.abc import Iterable import logging -from pyomo.common.collections import ComponentSet from pyomo.common.config import ( ConfigDict, ConfigValue, @@ -14,7 +12,6 @@ NonNegativeFloat, InEnum, Path, - _domain_name, ) from pyomo.common.errors import ApplicationError, PyomoException from pyomo.core.base import Var, VarData @@ -61,56 +58,56 @@ def positive_int_or_minus_one(obj): positive_int_or_minus_one.domain_name = "positive int or -1" -def mutable_param_validator(param_obj): +def uncertain_param_validator(uncertain_obj): """ - Check that Param-like object has attribute `mutable=True`. - - Parameters - ---------- - param_obj : Param or ParamData - Param-like object of interest. + Check that a component/component data object modeling an + uncertain parameter in PyROS is appropriately constructed, + initialized, and/or mutable, where applicable. Raises ------ ValueError - If lengths of the param object and the accompanying - index set do not match. This may occur if some entry - of the Param is not initialized. - ValueError - If attribute `mutable` is of value False. + If the length of the component (data) object does not + match that of its index set, or the object is an instance + of Param/ParamData with attribute `mutable=False`. """ - if len(param_obj) != len(param_obj.index_set()): + if len(uncertain_obj) != len(uncertain_obj.index_set()): raise ValueError( - f"Length of Param component object with " - f"name {param_obj.name!r} is {len(param_obj)}, " + f"Length of {type(uncertain_obj).__name__} object with " + f"name {uncertain_obj.name!r} is {len(uncertain_obj)}, " "and does not match that of its index set, " - f"which is of length {len(param_obj.index_set())}. " - "Check that all entries of the component object " - "have been initialized." + f"which is of length {len(uncertain_obj.index_set())}. " + "Check that the component has been properly constructed, " + "and all entries have been initialized. " + ) + if isinstance(uncertain_obj, (Param, ParamData)) and not uncertain_obj.mutable: + raise ValueError( + f"{type(uncertain_obj).__name__} object with name {uncertain_obj.name!r} " + "is immutable." ) - if not param_obj.mutable: - raise ValueError(f"Param object with name {param_obj.name!r} is immutable.") class InputDataStandardizer(object): """ - Standardizer for objects castable to a list of Pyomo - component types. + Domain validator for an object that is castable to + a list of Pyomo component data objects. Parameters ---------- - ctype : type - Pyomo component type, such as Component, Var or Param. - cdatatype : type - Corresponding Pyomo component data type, such as + ctype : type or tuple of type + Valid Pyomo component type(s), + such as Component, Var or Param. + cdatatype : type or tuple of type + Valid Pyomo component data type(s), such as ComponentData, VarData, or ParamData. ctype_validator : callable, optional Validator function for objects of type `ctype`. cdatatype_validator : callable, optional Validator function for objects of type `cdatatype`. allow_repeats : bool, optional - True to allow duplicate component data entries in final - list to which argument is cast, False otherwise. + True to allow duplicate component data object + entries in final list to which argument is cast, + False otherwise. Attributes ---------- @@ -151,13 +148,10 @@ def __call__(self, obj, from_iterable=None, allow_repeats=None): True if list can contain repeated entries, False otherwise. - Raises - ------ - TypeError - If all entries in the resulting list - are not of type ``self.cdatatype``. - ValueError - If the resulting list contains duplicate entries. + Returns + ------- + list of ComponentData + Each entry is an instance of ``self.cdatatype``. """ return standardize_component_data( obj=obj, @@ -171,9 +165,12 @@ def __call__(self, obj, from_iterable=None, allow_repeats=None): def domain_name(self): """Return str briefly describing domain encompassed by self.""" - cdt = _domain_name(self.cdatatype) - ct = _domain_name(self.ctype) - return f"{cdt}, {ct}, or Iterable[{cdt}/{ct}]" + ctypes_tup = (self.ctype,) if isinstance(self.ctype, type) else self.ctype + cdtypes_tup = ( + (self.cdatatype,) if isinstance(self.cdatatype, type) else self.cdatatype + ) + alltypes_desc = ", ".join(vtype.__name__ for vtype in ctypes_tup + cdtypes_tup) + return f"(iterable of) {alltypes_desc}" class SolverNotResolvable(PyomoException): @@ -508,7 +505,7 @@ def pyros_config(): domain=InputDataStandardizer( ctype=Param, cdatatype=ParamData, - ctype_validator=mutable_param_validator, + ctype_validator=uncertain_param_validator, allow_repeats=False, ), description=( diff --git a/pyomo/contrib/pyros/tests/test_config.py b/pyomo/contrib/pyros/tests/test_config.py index e4588953eca..5fc90c26d55 100644 --- a/pyomo/contrib/pyros/tests/test_config.py +++ b/pyomo/contrib/pyros/tests/test_config.py @@ -22,7 +22,7 @@ from pyomo.core.base.param import Param, ParamData from pyomo.contrib.pyros.config import ( InputDataStandardizer, - mutable_param_validator, + uncertain_param_validator, logger_domain, SolverNotResolvable, positive_int_or_minus_one, @@ -32,8 +32,6 @@ ) from pyomo.contrib.pyros.util import ObjectiveType from pyomo.opt import SolverFactory, SolverResults -from pyomo.contrib.pyros.uncertainty_sets import BoxSet -from pyomo.common.dependencies import numpy_available class TestInputDataStandardizer(unittest.TestCase): @@ -212,7 +210,7 @@ def test_standardizer_invalid_uninitialized_params(self): uninitialized entries passed. """ standardizer_func = InputDataStandardizer( - ctype=Param, cdatatype=ParamData, ctype_validator=mutable_param_validator + ctype=Param, cdatatype=ParamData, ctype_validator=uncertain_param_validator ) mdl = ConcreteModel() @@ -228,7 +226,7 @@ def test_standardizer_invalid_immutable_params(self): Param object(s) passed. """ standardizer_func = InputDataStandardizer( - ctype=Param, cdatatype=ParamData, ctype_validator=mutable_param_validator + ctype=Param, cdatatype=ParamData, ctype_validator=uncertain_param_validator ) mdl = ConcreteModel() @@ -238,6 +236,19 @@ def test_standardizer_invalid_immutable_params(self): with self.assertRaisesRegex(ValueError, exc_str): standardizer_func(mdl.p) + def test_standardizer_invalid_vars_not_constructed(self): + """ + Test standardizer with uncertain param validator + raises exception when Var that is not constructed is passed. + """ + standardizer_func = InputDataStandardizer( + ctype=Var, cdatatype=VarData, ctype_validator=uncertain_param_validator + ) + bad_var = Var() + exc_str = r"Length of .*does not match that of.*index set" + with self.assertRaisesRegex(ValueError, exc_str): + standardizer_func(bad_var) + def test_standardizer_valid_mutable_params(self): """ Test Param-like standardizer works as expected for sequence @@ -248,7 +259,7 @@ def test_standardizer_valid_mutable_params(self): mdl.p2 = Param(["a", "b"], initialize=1, mutable=True) standardizer_func = InputDataStandardizer( - ctype=Param, cdatatype=ParamData, ctype_validator=mutable_param_validator + ctype=Param, cdatatype=ParamData, ctype_validator=uncertain_param_validator ) standardizer_input = [mdl.p1[0], mdl.p2] @@ -281,6 +292,62 @@ def test_standardizer_valid_mutable_params(self): ), ) + def test_standardizer_multiple_ctypes_with_validator(self): + """ + Test input data standardizer when there are + multiple component/component data types. + """ + mdl = ConcreteModel() + mdl.p = Param([0, 1], initialize=0, mutable=True) + mdl.v = Var(["a", "b"], initialize=1) + + standardizer_func = InputDataStandardizer( + ctype=(Var, Param), + cdatatype=(VarData, ParamData), + ctype_validator=uncertain_param_validator, + ) + standardizer_input = [mdl.p, mdl.v] + standardizer_output = standardizer_func(standardizer_input) + expected_standardizer_output = [mdl.p[0], mdl.p[1], mdl.v["a"], mdl.v["b"]] + + self.assertIsInstance( + standardizer_output, + list, + msg=( + "Standardized output should be of type list, " + f"but is of type {standardizer_output.__class__.__name__}." + ), + ) + self.assertEqual( + len(standardizer_output), + len(expected_standardizer_output), + msg="Length of standardizer output is not as expected.", + ) + enum_zip = enumerate(zip(expected_standardizer_output, standardizer_output)) + for idx, (input, output) in enum_zip: + self.assertIs( + input, + output, + msg=( + f"Entry {input} (id {id(input)}) " + "is not identical to " + f"input component data object {output} " + f"(id {id(output)})" + ), + ) + + def test_standardizer_domain_name(self): + """ + Test domain name function works as expected. + """ + std1 = InputDataStandardizer(ctype=Param, cdatatype=ParamData) + self.assertEqual(std1.domain_name(), "(iterable of) Param, ParamData") + + std2 = InputDataStandardizer(ctype=(Param, Var), cdatatype=(ParamData, VarData)) + self.assertEqual( + std2.domain_name(), "(iterable of) Param, Var, ParamData, VarData" + ) + AVAILABLE_SOLVER_TYPE_NAME = "available_pyros_test_solver" From 7df8dec5796e26b7d94b9771634ebe5f417f3f64 Mon Sep 17 00:00:00 2001 From: jasherma Date: Fri, 29 Nov 2024 23:52:31 -0500 Subject: [PATCH 02/36] Allow PyROS to accept `Var`s as uncertain params --- pyomo/contrib/pyros/config.py | 25 ++-- pyomo/contrib/pyros/pyros.py | 6 +- pyomo/contrib/pyros/tests/test_grcs.py | 116 +++++++++++++++++- pyomo/contrib/pyros/util.py | 156 +++++++++++++++++++++++-- 4 files changed, 279 insertions(+), 24 deletions(-) diff --git a/pyomo/contrib/pyros/config.py b/pyomo/contrib/pyros/config.py index 3f82c2e3ae8..90a51087da7 100644 --- a/pyomo/contrib/pyros/config.py +++ b/pyomo/contrib/pyros/config.py @@ -60,16 +60,21 @@ def positive_int_or_minus_one(obj): def uncertain_param_validator(uncertain_obj): """ - Check that a component/component data object modeling an + Check that a component object modeling an uncertain parameter in PyROS is appropriately constructed, initialized, and/or mutable, where applicable. + Parameters + ---------- + uncertain_obj : Param or Var + Object on which to perform checks. + Raises ------ ValueError If the length of the component (data) object does not - match that of its index set, or the object is an instance - of Param/ParamData with attribute `mutable=False`. + match that of its index set, or the object is a Param + with attribute `mutable=False`. """ if len(uncertain_obj) != len(uncertain_obj.index_set()): raise ValueError( @@ -111,11 +116,11 @@ class InputDataStandardizer(object): Attributes ---------- - ctype - cdatatype - ctype_validator - cdatatype_validator - allow_repeats + ctype : type or tuple of type + cdatatype : type or tuple of type + ctype_validator : callable or None + cdatatype_validator : callable or None + allow_repeats : bool """ def __init__( @@ -503,8 +508,8 @@ def pyros_config(): ConfigValue( default=[], domain=InputDataStandardizer( - ctype=Param, - cdatatype=ParamData, + ctype=(Param, Var), + cdatatype=(ParamData, VarData), ctype_validator=uncertain_param_validator, allow_repeats=False, ), diff --git a/pyomo/contrib/pyros/pyros.py b/pyomo/contrib/pyros/pyros.py index 2ffef5054aa..18728acc636 100644 --- a/pyomo/contrib/pyros/pyros.py +++ b/pyomo/contrib/pyros/pyros.py @@ -299,10 +299,10 @@ def solve( First-stage model variables (or design variables). second_stage_variables: VarData, Var, or iterable of VarData/Var Second-stage model variables (or control variables). - uncertain_params: ParamData, Param, or iterable of ParamData/Param + uncertain_params: (iterable of) Param, Var, ParamData, or VarData Uncertain model parameters. - The `mutable` attribute for all uncertain parameter objects - must be set to True. + Of every constituent `Param` and `ParamData` object, + the `mutable` attribute must be set to True. uncertainty_set: UncertaintySet Uncertainty set against which the solution(s) returned will be confirmed to be robust. diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index ebb2c8b7a37..4383ed45602 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -24,9 +24,14 @@ from pyomo.core.base.set_types import NonNegativeIntegers from pyomo.repn.plugins import nl_writer as pyomo_nl_writer import pyomo.repn.ampl as pyomo_ampl_repn -from pyomo.common.dependencies import numpy as np, numpy_available -from pyomo.common.dependencies import scipy_available +from pyomo.common.dependencies import ( + attempt_import, + numpy as np, + numpy_available, + scipy_available, +) from pyomo.common.errors import ApplicationError, InfeasibleConstraintException +from pyomo.core.expr import replace_expressions from pyomo.environ import maximize as pyo_max, units as u from pyomo.opt import ( SolverResults, @@ -69,10 +74,13 @@ logger = logging.getLogger(__name__) +parameterized, param_available = attempt_import('parameterized') -if not (numpy_available and scipy_available): +if not (numpy_available and scipy_available and param_available): raise unittest.SkipTest('PyROS unit tests require parameterized, numpy, and scipy') +parameterized = parameterized.parameterized + # === Config args for testing nlp_solver = 'ipopt' global_solver = 'baron' @@ -1704,6 +1712,55 @@ def test_coefficient_matching_nonlinear_expr(self): pyrosTerminationCondition.robust_feasible, ) + @unittest.skipUnless(ipopt_available, "IPOPT not available.") + def test_pyros_vars_as_uncertain_params(self): + """ + Test PyROS solver result is invariant to the type used + in argument `uncertain_params`. + """ + mdl1 = build_leyffer() + + # clone: use a Var to represent the uncertain parameter + mdl2 = mdl1.clone() + mdl2.uvar = Var(initialize=mdl2.u.value) + mdl2.con.set_value( + replace_expressions( + expr=mdl2.con.expr, substitution_map={id(mdl2.u): mdl2.uvar} + ) + ) + + box_set = BoxSet([[0.25, 2]]) + ipopt_solver = SolverFactory("ipopt") + pyros_solver = SolverFactory("pyros") + + res1 = pyros_solver.solve( + model=mdl1, + first_stage_variables=[mdl1.x1, mdl1.x2], + second_stage_variables=[], + uncertain_params=[mdl1.u], + uncertainty_set=box_set, + local_solver=ipopt_solver, + global_solver=ipopt_solver, + ) + self.assertEqual( + res1.pyros_termination_condition, pyrosTerminationCondition.robust_feasible + ) + + res2 = pyros_solver.solve( + model=mdl2, + first_stage_variables=[mdl2.x1, mdl2.x2], + second_stage_variables=[], + uncertain_params=[mdl2.uvar], + uncertainty_set=box_set, + local_solver=ipopt_solver, + global_solver=ipopt_solver, + ) + self.assertEqual( + res2.pyros_termination_condition, res1.pyros_termination_condition + ) + self.assertEqual(res1.final_objective_value, res2.final_objective_value) + self.assertEqual(res1.iterations, res2.iterations) + @unittest.skipUnless(scip_available, "Global NLP solver is not available.") class testBypassingSeparation(unittest.TestCase): @@ -2920,6 +2977,59 @@ def test_pyros_overlap_dof_vars(self): expected_regex="Ensure no Vars are included in both arguments.", ) + @parameterized.expand([["first_stage", True], ["second_stage", False]]) + def test_pyros_overlap_uncertain_params_vars(self, stage_name, is_first_stage): + """ + Test PyROS solver raises exception if there + is overlap between `uncertain_params` and either + `first_stage_variables` or `second_stage_variables`. + """ + # build model + mdl = self.build_simple_test_model() + + first_stage_vars = [mdl.x1, mdl.x2] if is_first_stage else [] + second_stage_vars = [mdl.x1, mdl.x2] if not is_first_stage else [] + + # prepare solvers + pyros = SolverFactory("pyros") + local_solver = SimpleTestSolver() + global_solver = SimpleTestSolver() + + # perform checks + exc_str = ( + f"Arguments `{stage_name}_variables` and `uncertain_params` " + "contain at least one common Var object." + ) + with LoggingIntercept(level=logging.ERROR) as LOG: + with self.assertRaisesRegex(ValueError, exc_str): + pyros.solve( + model=mdl, + first_stage_variables=first_stage_vars, + second_stage_variables=second_stage_vars, + uncertain_params=[mdl.x1], + uncertainty_set=BoxSet([[1 / 4, 2]]), + local_solver=local_solver, + global_solver=global_solver, + ) + + # check logger output is as expected + log_msgs = LOG.getvalue().split("\n")[:-1] + self.assertEqual( + len(log_msgs), 3, "Error message does not contain expected number of lines." + ) + self.assertRegex( + text=log_msgs[0], + expected_regex=( + f"The following Vars were found in both `{stage_name}_variables`" + "and `uncertain_params`.*" + ), + ) + self.assertRegex(text=log_msgs[1], expected_regex=" 'x1'") + self.assertRegex( + text=log_msgs[2], + expected_regex="Ensure no Vars are included in both arguments.", + ) + def test_pyros_vars_not_in_model(self): """ Test PyROS appropriately raises exception if there are diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index e95be5b7b30..16edf4cd935 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -39,8 +39,10 @@ Objective, maximize, minimize, + Param, Reals, Var, + VarData, value, ) from pyomo.core.expr.numeric_expr import SumExpression @@ -558,7 +560,44 @@ def standardize_component_data( from_iterable=None, ): """ - Standardize object to a list of component data objects. + Cast an object to a list of Pyomo ComponentData objects. + + Parameters + ---------- + obj : Component, ComponentData, or iterable + Object from which component data objects + are cast. + valid_ctype : type or tuple of type + Valid Component type(s). + valid_cdatatype : type or tuple of type + Valid ComponentData type(s). + ctype_validator : None or callable, optional + Validator for component objects derived from `obj`. + cdatatype_validator : None or callable, optional + Validator for component data objects derived from `obj`. + allow_repeats : bool, optional + True to allow for nonunique component data objects + derived from `obj`, False otherwise. + from_iterable : str, optional + Description of the object to include in error messages. + Meant to be used if the object is an iterable from which + to derive component data objects. + + Returns + ------- + list of ComponentData + The ComponentData objects derived from `obj`. + Note: If `obj` is a valid ComponentData type, + then ``[obj]`` is returned. + + Raises + ------ + TypeError + If `obj` is not an iterable and not an instance of + `valid_ctype` or `valid_cdatatype`. + ValueError + If ``allow_repeats=False`` and there are duplicates + among the component data objects derived from `obj`. """ if isinstance(obj, valid_ctype): if ctype_validator is not None: @@ -745,9 +784,8 @@ def validate_model(model, config): def validate_variable_partitioning(model, config): """ - Check that partitioning of the first-stage variables, - second-stage variables, and uncertain parameters - is valid. + Check that the partitioning of the in-scope variables of the + model is valid. Parameters ---------- @@ -791,6 +829,8 @@ def validate_variable_partitioning(model, config): "contain at least one common Var object." ) + # uncertain parameters can be VarData objects; + # ensure they are not considered decision variables here active_model_vars = ComponentSet( get_vars_from_components( block=model, @@ -799,7 +839,7 @@ def validate_variable_partitioning(model, config): descend_into=True, ctype=(Objective, Constraint), ) - ) + ) - ComponentSet(config.uncertain_params) check_components_descended_from_model( model=model, components=active_model_vars, @@ -837,6 +877,9 @@ def validate_uncertainty_specification(model, config): ValueError If at least one of the following holds: + - there are entries of `config.uncertain_params` + that are also in `config.first_stage_variables` or + `config.second_stage_variables` - dimension of uncertainty set does not equal number of uncertain parameters - uncertainty set `is_valid()` method does not return @@ -850,6 +893,26 @@ def validate_uncertainty_specification(model, config): config=config, ) + first_stg_vars = config.first_stage_variables + second_stg_vars = config.second_stage_variables + for stg_str, vars in zip(["first", "second"], [first_stg_vars, second_stg_vars]): + overlapping_uncertain_params = ComponentSet(vars) & ComponentSet( + config.uncertain_params + ) + if overlapping_uncertain_params: + overlapping_var_list = "\n ".join( + f"{var.name!r}" for var in overlapping_uncertain_params + ) + config.progress_logger.error( + f"The following Vars were found in both `{stg_str}_stage_variables`" + f"and `uncertain_params`:\n {overlapping_var_list}" + "\nEnsure no Vars are included in both arguments." + ) + raise ValueError( + f"Arguments `{stg_str}_stage_variables` and `uncertain_params` " + "contain at least one common Var object." + ) + if len(config.uncertain_params) != config.uncertainty_set.dim: raise ValueError( "Length of argument `uncertain_params` does not match dimension " @@ -1598,6 +1661,44 @@ def turn_adjustable_var_bounds_to_constraints(model_data): # the interface for separation priority ordering +def replace_vars_with_params(block, var_to_param_map): + """ + Substitute ParamData objects for VarData objects + in all named expression, constraint, and objective components + declared on a block and all its sub-blocks. + + Named Expressions are removed from Constraint and Objective + components during the substitution. + + Parameters + ---------- + block : BlockData + Block on which to perform the substitution. + var_to_param_map : ComponentMap + Mapping from VarData objects to be replaced + to the ParamData objects to be introduced. + """ + substitution_map = {id(var): param for var, param in var_to_param_map.items()} + vars_to_replace = ComponentSet(var_to_param_map.keys()) + ctypes = (Constraint, Objective, Expression) + cdata_objs = ( + cdata + for cdata in block.component_data_objects( + ctype=ctypes, active=None, descend_into=True + ) + if ComponentSet(identify_variables(cdata.expr)) & vars_to_replace + ) + for cdata in cdata_objs: + cdata.set_value( + replace_expressions( + expr=cdata.expr, + substitution_map=substitution_map, + descend_into_named_expressions=True, + remove_named_expressions=True, + ) + ) + + def setup_working_model(model_data, user_var_partitioning): """ Set up (construct) the working model based on user inputs, @@ -1619,7 +1720,7 @@ def setup_working_model(model_data, user_var_partitioning): temp_util_block_attr_name = unique_component_name(original_model, "util") original_model.add_component(temp_util_block_attr_name, Block()) orig_temp_util_block = getattr(original_model, temp_util_block_attr_name) - orig_temp_util_block.uncertain_params = config.uncertain_params + orig_temp_util_block.orig_uncertain_params = config.uncertain_params orig_temp_util_block.user_var_partitioning = VariablePartitioning( **user_var_partitioning._asdict() ) @@ -1643,8 +1744,8 @@ def setup_working_model(model_data, user_var_partitioning): working_temp_util_block = getattr( working_model.user_model, temp_util_block_attr_name ) - model_data.working_model.uncertain_params = ( - working_temp_util_block.uncertain_params.copy() + model_data.working_model.orig_uncertain_params = ( + working_temp_util_block.orig_uncertain_params.copy() ) working_model.user_var_partitioning = VariablePartitioning( **working_temp_util_block.user_var_partitioning._asdict() @@ -1654,6 +1755,45 @@ def setup_working_model(model_data, user_var_partitioning): delattr(original_model, temp_util_block_attr_name) delattr(working_model.user_model, temp_util_block_attr_name) + uncertain_param_var_idxs = [] + for idx, obj in enumerate(working_model.orig_uncertain_params): + if isinstance(obj, VarData): + config.progress_logger.debug( + "Entry of argument `uncertain_params` with name " + f"{obj.name!r} is of type {VarData.__name__}. " + f"Bounds and fixing for this {VarData.__name__} object " + "will be ignored. " + "A temporary ParamData object will be substituted for " + f"{obj.name!r} in all expressions, constraints, and objectives " + "of the working model clone. " + ) + obj.fix() + uncertain_param_var_idxs.append(idx) + temp_params = working_model.temp_uncertain_params = Param( + uncertain_param_var_idxs, + within=Reals, + initialize={ + idx: config.nominal_uncertain_param_vals[idx] + for idx in uncertain_param_var_idxs + }, + mutable=True, + ) + working_model.uncertain_params = [ + temp_params[idx] if idx in uncertain_param_var_idxs else orig_param + for idx, orig_param in enumerate(working_model.orig_uncertain_params) + ] + + # don't want to pass over the model components unless + # at least one Var is to be replaced + if uncertain_param_var_idxs: + replace_vars_with_params( + working_model, + var_to_param_map=ComponentMap( + (working_model.orig_uncertain_params[idx], temp_param) + for idx, temp_param in temp_params.items() + ), + ) + # keep track of the original active constraints working_model.original_active_equality_cons = [] working_model.original_active_inequality_cons = [] From 24f0c99a5f5a19ca2d3ca1807d29aa9b89c6fdf5 Mon Sep 17 00:00:00 2001 From: jasherma Date: Sat, 30 Nov 2024 11:27:30 -0500 Subject: [PATCH 03/36] Test uncertain param substitution in working model startup --- .../contrib/pyros/tests/test_preprocessor.py | 94 ++++++++++++++++++- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/pyros/tests/test_preprocessor.py b/pyomo/contrib/pyros/tests/test_preprocessor.py index 0f5f131a4a5..f0f75306397 100644 --- a/pyomo/contrib/pyros/tests/test_preprocessor.py +++ b/pyomo/contrib/pyros/tests/test_preprocessor.py @@ -27,6 +27,7 @@ Any, Var, Constraint, + Expression, Objective, ConcreteModel, Param, @@ -35,7 +36,7 @@ Block, ) from pyomo.core.base.set_types import NonNegativeReals, NonPositiveReals, Reals -from pyomo.core.expr import log, sin, exp, RangedExpression +from pyomo.core.expr import LinearExpression, log, replace_expressions, sin, exp, RangedExpression from pyomo.core.expr.compare import assertExpressionsEqual from pyomo.contrib.pyros.util import ( @@ -348,10 +349,18 @@ def build_test_model_data(self): # Objective and Constraint objects m.y3 = Var(domain=RangeSet(0, 1, 0), bounds=(0.2, 0.5)) + # Var to represent an uncertain Param; + # bounds will be ignored + m.q2var = Var(bounds=(0, None), initialize=3.2) + # fix some variables m.z4.fix() m.y2.fix() + # NAMED EXPRESSIONS: mainly to test + # Var -> Param substitution for uncertain params + m.nexpr = Expression(expr=m.q2var + m.x1) + # EQUALITY CONSTRAINTS m.eq1 = Constraint(expr=m.q * (m.z3 + m.x2) == 0) m.eq2 = Constraint(expr=m.x1 - m.z1 == 0) @@ -362,7 +371,7 @@ def build_test_model_data(self): m.ineq1 = Constraint(expr=(-m.p, m.x1 + m.z1, exp(m.q))) m.ineq2 = Constraint(expr=(0, m.x1 + m.x2, 10)) m.ineq3 = Constraint(expr=(2 * m.q, 2 * (m.z3 + m.y1), 2 * m.q)) - m.ineq4 = Constraint(expr=-m.q <= m.y2**2 + log(m.y2)) + m.ineq4 = Constraint(expr=-m.q <= m.y2**2 + log(m.y2) + m.q2var) # out of scope: deactivated m.ineq5 = Constraint(expr=m.y3 <= m.q) @@ -383,6 +392,10 @@ def build_test_model_data(self): ) ) + # inactive objective + m.inactive_obj = Objective(expr=1 + m.q2var + m.x1) + m.inactive_obj.deactivate() + # set up the var partitioning user_var_partitioning = VariablePartitioning( first_stage_variables=[m.x1, m.x2], @@ -400,7 +413,9 @@ def test_setup_working_model(self): model_data, user_var_partitioning = self.build_test_model_data() om = model_data.original_model config = model_data.config - config.uncertain_params = [om.q] + config.uncertain_params = [om.q, om.q2var] + config.progress_logger = logger + config.nominal_uncertain_param_vals = [om.q.value, om.q2var.value] setup_working_model(model_data, user_var_partitioning) working_model = model_data.working_model @@ -418,6 +433,7 @@ def test_setup_working_model(self): # active objective self.assertTrue(m.obj.active) + self.assertFalse(m.inactive_obj.active) # user var partitioning up = working_model.user_var_partitioning @@ -432,7 +448,15 @@ def test_setup_working_model(self): # uncertain params self.assertEqual( - ComponentSet(working_model.uncertain_params), ComponentSet([m.q]) + ComponentSet(working_model.orig_uncertain_params), + ComponentSet([m.q, m.q2var]) + ) + + self.assertEqual(list(working_model.temp_uncertain_params.index_set()), [1]) + temp_uncertain_param = working_model.temp_uncertain_params[1] + self.assertEqual( + ComponentSet(working_model.uncertain_params), + ComponentSet([m.q, temp_uncertain_param]) ) # ensure original model unchanged @@ -446,6 +470,68 @@ def test_setup_working_model(self): self.assertFalse(working_model.second_stage.inequality_cons) self.assertFalse(working_model.second_stage.equality_cons) + # ensure uncertain Param substitutions carried out properly + ublk = model_data.working_model.user_model + self.assertExpressionsEqual( + ublk.nexpr.expr, + temp_uncertain_param + ublk.x1, + ) + self.assertExpressionsEqual( + ublk.inactive_obj.expr, + LinearExpression([1, temp_uncertain_param, m.x1]), + ) + self.assertExpressionsEqual( + ublk.ineq4.expr, + -ublk.q <= ublk.y2 ** 2 + log(ublk.y2) + temp_uncertain_param, + ) + + # other component expressions should remain as declared + self.assertExpressionsEqual( + ublk.eq1.expr, + ublk.q * (ublk.z3 + ublk.x2) == 0, + ) + self.assertExpressionsEqual( + ublk.eq2.expr, + ublk.x1 - ublk.z1 == 0, + ) + self.assertExpressionsEqual( + ublk.eq3.expr, + ublk.x1 ** 2 + ublk.x2 + ublk.p * ublk.z2 == ublk.p, + ) + self.assertExpressionsEqual( + ublk.eq4.expr, + ublk.z3 + ublk.y1 == ublk.q, + ) + self.assertExpressionsEqual( + ublk.ineq1.expr, + RangedExpression((-ublk.p, ublk.x1 + ublk.z1, exp(ublk.q)), False), + ) + self.assertExpressionsEqual( + ublk.ineq2.expr, + RangedExpression((0, ublk.x1 + ublk.x2, 10), False), + ) + self.assertExpressionsEqual( + ublk.ineq3.expr, + RangedExpression((2 * ublk.q, 2 * (ublk.z3 + ublk.y1), 2 * ublk.q), False), + ) + self.assertExpressionsEqual( + ublk.ineq5.expr, + ublk.y3 <= ublk.q, + ) + self.assertExpressionsEqual( + ublk.obj.expr, + ( + ublk.p**2 + + 2 * ublk.p * ublk.q + + log(ublk.x1) + + 2 * ublk.p * ublk.x1 + + ublk.q**2 * ublk.x1 + + ublk.p**3 * (ublk.z1 + ublk.z2 + ublk.y1) + + ublk.z4 + + ublk.z5 + ) + ) + class TestResolveVarBounds(unittest.TestCase): """ From 40a134b6de945dfbe44acfbe890d5e8c32f0c4c7 Mon Sep 17 00:00:00 2001 From: jasherma Date: Sat, 30 Nov 2024 11:28:55 -0500 Subject: [PATCH 04/36] Apply black --- .../contrib/pyros/tests/test_preprocessor.py | 52 +++++++------------ 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/pyomo/contrib/pyros/tests/test_preprocessor.py b/pyomo/contrib/pyros/tests/test_preprocessor.py index f0f75306397..0241d89bab7 100644 --- a/pyomo/contrib/pyros/tests/test_preprocessor.py +++ b/pyomo/contrib/pyros/tests/test_preprocessor.py @@ -36,7 +36,14 @@ Block, ) from pyomo.core.base.set_types import NonNegativeReals, NonPositiveReals, Reals -from pyomo.core.expr import LinearExpression, log, replace_expressions, sin, exp, RangedExpression +from pyomo.core.expr import ( + LinearExpression, + log, + replace_expressions, + sin, + exp, + RangedExpression, +) from pyomo.core.expr.compare import assertExpressionsEqual from pyomo.contrib.pyros.util import ( @@ -449,14 +456,14 @@ def test_setup_working_model(self): # uncertain params self.assertEqual( ComponentSet(working_model.orig_uncertain_params), - ComponentSet([m.q, m.q2var]) + ComponentSet([m.q, m.q2var]), ) self.assertEqual(list(working_model.temp_uncertain_params.index_set()), [1]) temp_uncertain_param = working_model.temp_uncertain_params[1] self.assertEqual( ComponentSet(working_model.uncertain_params), - ComponentSet([m.q, temp_uncertain_param]) + ComponentSet([m.q, temp_uncertain_param]), ) # ensure original model unchanged @@ -472,52 +479,33 @@ def test_setup_working_model(self): # ensure uncertain Param substitutions carried out properly ublk = model_data.working_model.user_model + self.assertExpressionsEqual(ublk.nexpr.expr, temp_uncertain_param + ublk.x1) self.assertExpressionsEqual( - ublk.nexpr.expr, - temp_uncertain_param + ublk.x1, - ) - self.assertExpressionsEqual( - ublk.inactive_obj.expr, - LinearExpression([1, temp_uncertain_param, m.x1]), + ublk.inactive_obj.expr, LinearExpression([1, temp_uncertain_param, m.x1]) ) self.assertExpressionsEqual( - ublk.ineq4.expr, - -ublk.q <= ublk.y2 ** 2 + log(ublk.y2) + temp_uncertain_param, + ublk.ineq4.expr, -ublk.q <= ublk.y2**2 + log(ublk.y2) + temp_uncertain_param ) # other component expressions should remain as declared + self.assertExpressionsEqual(ublk.eq1.expr, ublk.q * (ublk.z3 + ublk.x2) == 0) + self.assertExpressionsEqual(ublk.eq2.expr, ublk.x1 - ublk.z1 == 0) self.assertExpressionsEqual( - ublk.eq1.expr, - ublk.q * (ublk.z3 + ublk.x2) == 0, - ) - self.assertExpressionsEqual( - ublk.eq2.expr, - ublk.x1 - ublk.z1 == 0, - ) - self.assertExpressionsEqual( - ublk.eq3.expr, - ublk.x1 ** 2 + ublk.x2 + ublk.p * ublk.z2 == ublk.p, - ) - self.assertExpressionsEqual( - ublk.eq4.expr, - ublk.z3 + ublk.y1 == ublk.q, + ublk.eq3.expr, ublk.x1**2 + ublk.x2 + ublk.p * ublk.z2 == ublk.p ) + self.assertExpressionsEqual(ublk.eq4.expr, ublk.z3 + ublk.y1 == ublk.q) self.assertExpressionsEqual( ublk.ineq1.expr, RangedExpression((-ublk.p, ublk.x1 + ublk.z1, exp(ublk.q)), False), ) self.assertExpressionsEqual( - ublk.ineq2.expr, - RangedExpression((0, ublk.x1 + ublk.x2, 10), False), + ublk.ineq2.expr, RangedExpression((0, ublk.x1 + ublk.x2, 10), False) ) self.assertExpressionsEqual( ublk.ineq3.expr, RangedExpression((2 * ublk.q, 2 * (ublk.z3 + ublk.y1), 2 * ublk.q), False), ) - self.assertExpressionsEqual( - ublk.ineq5.expr, - ublk.y3 <= ublk.q, - ) + self.assertExpressionsEqual(ublk.ineq5.expr, ublk.y3 <= ublk.q) self.assertExpressionsEqual( ublk.obj.expr, ( @@ -529,7 +517,7 @@ def test_setup_working_model(self): + ublk.p**3 * (ublk.z1 + ublk.z2 + ublk.y1) + ublk.z4 + ublk.z5 - ) + ), ) From 2b9bd290e3b98187308548b5936753bd8df6cfff Mon Sep 17 00:00:00 2001 From: jasherma Date: Sat, 30 Nov 2024 11:29:18 -0500 Subject: [PATCH 05/36] Remove unused import --- pyomo/contrib/pyros/tests/test_preprocessor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/contrib/pyros/tests/test_preprocessor.py b/pyomo/contrib/pyros/tests/test_preprocessor.py index 0241d89bab7..2c339f5904a 100644 --- a/pyomo/contrib/pyros/tests/test_preprocessor.py +++ b/pyomo/contrib/pyros/tests/test_preprocessor.py @@ -39,7 +39,6 @@ from pyomo.core.expr import ( LinearExpression, log, - replace_expressions, sin, exp, RangedExpression, From 90e4e1581e8bbe1cbf00b8294801b689a3cec53d Mon Sep 17 00:00:00 2001 From: jasherma Date: Sat, 30 Nov 2024 15:20:51 -0500 Subject: [PATCH 06/36] Account for `Var`-type uncertain params in preprocessor tests --- .../contrib/pyros/tests/test_preprocessor.py | 104 ++++++++++++++---- 1 file changed, 81 insertions(+), 23 deletions(-) diff --git a/pyomo/contrib/pyros/tests/test_preprocessor.py b/pyomo/contrib/pyros/tests/test_preprocessor.py index 2c339f5904a..e1aed527e9c 100644 --- a/pyomo/contrib/pyros/tests/test_preprocessor.py +++ b/pyomo/contrib/pyros/tests/test_preprocessor.py @@ -2218,6 +2218,13 @@ def build_test_model_data(self): m.z4.fix() m.y2.fix() + # Var representing uncertain parameter + m.q2var = Var(initialize=3.2) + + # named Expression in terms of uncertain parameter + # represented by a Var + m.q2expr = Expression(expr=m.q2var * 10) + # EQUALITY CONSTRAINTS # this will be reformulated by coefficient matching m.eq1 = Constraint(expr=m.q * (m.z3 + m.x2) == 0) @@ -2227,7 +2234,7 @@ def build_test_model_data(self): # pretriangular: makes z2 nonadjustable, so first-stage m.eq3 = Constraint(expr=m.x1**2 + m.x2 + m.p * m.z2 == m.p) # second-stage equality - m.eq4 = Constraint(expr=m.z3 + m.y1 == m.q) + m.eq4 = Constraint(expr=m.z3 + m.y1 + 5 * m.q2var == m.q) # INEQUALITY CONSTRAINTS # since x1, z1 nonadjustable, LB is first-stage, @@ -2250,6 +2257,11 @@ def build_test_model_data(self): m.ineq5 = Constraint(expr=m.y3 <= m.q) m.ineq5.deactivate() + # ineq constraint in which the only uncertain parameter + # is represented by a Var. will be second-stage due + # to the presence of the uncertain parameter + m.ineq6 = Constraint(expr=-m.q2var <= m.x1) + # OBJECTIVE # contains a rich combination of first-stage and second-stage terms m.obj = Objective( @@ -2262,6 +2274,7 @@ def build_test_model_data(self): + m.p**3 * (m.z1 + m.z2 + m.y1) + m.z4 + m.z5 + + m.q2expr ) ) @@ -2288,7 +2301,8 @@ def test_preprocessor_effective_var_partitioning_static_dr(self): config = model_data.config config.update( dict( - uncertain_params=[om.q], + uncertain_params=[om.q, om.q2var], + nominal_uncertain_param_vals=[om.q.value, om.q2var.value], objective_focus=ObjectiveType.worst_case, decision_rule_order=0, progress_logger=logger, @@ -2424,7 +2438,8 @@ def test_preprocessor_constraint_partitioning_nonstatic_dr( om = model_data.original_model model_data.config.update( dict( - uncertain_params=[om.q], + uncertain_params=[om.q, om.q2var], + nominal_uncertain_param_vals=[om.q.value, om.q2var.value], objective_focus=ObjectiveType[obj_focus], decision_rule_order=dr_order, progress_logger=logger, @@ -2443,14 +2458,19 @@ def test_preprocessor_constraint_partitioning_nonstatic_dr( coeff_matching_con_names = [ "coeff_matching_var_z5_uncertain_eq_bound_con_coeff_0", "coeff_matching_var_z5_uncertain_eq_bound_con_coeff_1", + "coeff_matching_var_z5_uncertain_eq_bound_con_coeff_2", 'coeff_matching_eq_con_eq1_coeff_1', 'coeff_matching_eq_con_eq1_coeff_2', + 'coeff_matching_eq_con_eq1_coeff_3', ] else: coeff_matching_con_names = [ "coeff_matching_var_z5_uncertain_eq_bound_con_coeff_0", "coeff_matching_var_z5_uncertain_eq_bound_con_coeff_1", "coeff_matching_var_z5_uncertain_eq_bound_con_coeff_2", + "coeff_matching_var_z5_uncertain_eq_bound_con_coeff_3", + "coeff_matching_var_z5_uncertain_eq_bound_con_coeff_4", + "coeff_matching_var_z5_uncertain_eq_bound_con_coeff_5", ] self.assertEqual( @@ -2479,6 +2499,7 @@ def test_preprocessor_constraint_partitioning_nonstatic_dr( "ineq_con_ineq3_lower_bound_con", "ineq_con_ineq3_upper_bound_con", "ineq_con_ineq4_lower_bound_con", + "ineq_con_ineq6_lower_bound_con", ] + (["epigraph_con"] if obj_focus == "worst_case" else []) + ( @@ -2578,6 +2599,11 @@ def test_preprocessor_constraint_partitioning_nonstatic_dr( -(m.y2**2 + log(m.y2)) <= -(-m.q), ) self.assertFalse(m.ineq5.active) + assertExpressionsEqual( + self, + ss.inequality_cons["ineq_con_ineq6_lower_bound_con"].expr, + -m.x1 <= - (-1 * working_model.temp_uncertain_params[1]), + ) assertExpressionsEqual( self, fs.equality_cons["eq_con_eq2"].expr, m.x1 - m.z1 == 0 @@ -2620,7 +2646,8 @@ def test_preprocessor_coefficient_matching( config = model_data.config config.update( dict( - uncertain_params=[om.q], + uncertain_params=[om.q, om.q2var], + nominal_uncertain_param_vals=[om.q.value, om.q2var.value], objective_focus=ObjectiveType.worst_case, decision_rule_order=dr_order, progress_logger=logger, @@ -2653,6 +2680,11 @@ def test_preprocessor_coefficient_matching( fs_eqs["coeff_matching_var_z5_uncertain_eq_bound_con_coeff_1"].expr, fs.decision_rule_vars[1][1] - 1 == 0, ) + assertExpressionsEqual( + self, + fs_eqs["coeff_matching_var_z5_uncertain_eq_bound_con_coeff_2"].expr, + fs.decision_rule_vars[1][2] == 0, + ) assertExpressionsEqual( self, fs_eqs["coeff_matching_eq_con_eq1_coeff_1"].expr, @@ -2663,6 +2695,11 @@ def test_preprocessor_coefficient_matching( fs_eqs["coeff_matching_eq_con_eq1_coeff_2"].expr, fs.decision_rule_vars[0][1] == 0, ) + assertExpressionsEqual( + self, + fs_eqs["coeff_matching_eq_con_eq1_coeff_3"].expr, + fs.decision_rule_vars[0][2] == 0, + ) if config.decision_rule_order == 2: # eq1 should be deactivated and refomulated to 2 inequalities assertExpressionsEqual( @@ -2692,6 +2729,21 @@ def test_preprocessor_coefficient_matching( fs_eqs["coeff_matching_var_z5_uncertain_eq_bound_con_coeff_2"].expr, fs.decision_rule_vars[1][2] == 0, ) + assertExpressionsEqual( + self, + fs_eqs["coeff_matching_var_z5_uncertain_eq_bound_con_coeff_3"].expr, + fs.decision_rule_vars[1][3] == 0, + ) + assertExpressionsEqual( + self, + fs_eqs["coeff_matching_var_z5_uncertain_eq_bound_con_coeff_4"].expr, + fs.decision_rule_vars[1][4] == 0, + ) + assertExpressionsEqual( + self, + fs_eqs["coeff_matching_var_z5_uncertain_eq_bound_con_coeff_5"].expr, + fs.decision_rule_vars[1][5] == 0, + ) @parameterized.expand([["static", 0], ["affine", 1], ["quadratic", 2]]) def test_preprocessor_objective_standardization(self, name, dr_order): @@ -2704,7 +2756,8 @@ def test_preprocessor_objective_standardization(self, name, dr_order): config = model_data.config config.update( dict( - uncertain_params=[om.q], + uncertain_params=[om.q, om.q2var], + nominal_uncertain_param_vals=[om.q.value, om.q2var.value], objective_focus=ObjectiveType.worst_case, decision_rule_order=dr_order, progress_logger=logger, @@ -2742,6 +2795,9 @@ def test_preprocessor_objective_standardization(self, name, dr_order): + ublk.p**3 * (ublk.z1 + ublk.z2 + ublk.y1) + ublk.z4 + ublk.z5 + # TODO: are we sure this shouldn't be q2expr.expr? + # expected removal of named expression + + ublk.q2expr ), ) @@ -2756,7 +2812,8 @@ def test_preprocessor_log_model_statistics_affine_dr(self, obj_focus): config = model_data.config config.update( dict( - uncertain_params=[om.q], + uncertain_params=[om.q, om.q2var], + nominal_uncertain_param_vals=[om.q.value, om.q2var.value], objective_focus=ObjectiveType[obj_focus], decision_rule_order=1, progress_logger=logger, @@ -2769,22 +2826,22 @@ def test_preprocessor_log_model_statistics_affine_dr(self, obj_focus): expected_log_str = textwrap.dedent( f""" Model Statistics: - Number of variables : 14 + Number of variables : 16 Epigraph variable : 1 First-stage variables : 2 Second-stage variables : 5 (2 adj.) State variables : 2 (1 adj.) - Decision rule variables : 4 - Number of uncertain parameters : 1 - Number of constraints : 23 - Equality constraints : 9 - Coefficient matching constraints : 4 + Decision rule variables : 6 + Number of uncertain parameters : 2 + Number of constraints : 26 + Equality constraints : 11 + Coefficient matching constraints : 6 Other first-stage equations : 2 Second-stage equations : 1 Decision rule equations : 2 - Inequality constraints : 14 + Inequality constraints : 15 First-stage inequalities : {3 if obj_focus == 'nominal' else 2} - Second-stage inequalities : {11 if obj_focus == 'nominal' else 12} + Second-stage inequalities : {12 if obj_focus == 'nominal' else 13} """ ) @@ -2810,7 +2867,8 @@ def test_preprocessor_log_model_statistics_quadratic_dr(self, obj_focus): config = model_data.config config.update( dict( - uncertain_params=[om.q], + uncertain_params=[om.q, om.q2var], + nominal_uncertain_param_vals=[om.q.value, om.q2var.value], objective_focus=ObjectiveType[obj_focus], decision_rule_order=2, progress_logger=logger, @@ -2823,22 +2881,22 @@ def test_preprocessor_log_model_statistics_quadratic_dr(self, obj_focus): expected_log_str = textwrap.dedent( f""" Model Statistics: - Number of variables : 16 + Number of variables : 22 Epigraph variable : 1 First-stage variables : 2 Second-stage variables : 5 (2 adj.) State variables : 2 (1 adj.) - Decision rule variables : 6 - Number of uncertain parameters : 1 - Number of constraints : 24 - Equality constraints : 8 - Coefficient matching constraints : 3 + Decision rule variables : 12 + Number of uncertain parameters : 2 + Number of constraints : 28 + Equality constraints : 11 + Coefficient matching constraints : 6 Other first-stage equations : 2 Second-stage equations : 1 Decision rule equations : 2 - Inequality constraints : 16 + Inequality constraints : 17 First-stage inequalities : {3 if obj_focus == 'nominal' else 2} - Second-stage inequalities : {13 if obj_focus == 'nominal' else 14} + Second-stage inequalities : {14 if obj_focus == 'nominal' else 15} """ ) From 3d7b8f0d61ce7ac7069c1ff992d6c29c19361c5c Mon Sep 17 00:00:00 2001 From: jasherma Date: Sat, 30 Nov 2024 16:05:43 -0500 Subject: [PATCH 07/36] Maintain named expressions in `Param` for `Var` substitutions --- .../contrib/pyros/tests/test_preprocessor.py | 2 - pyomo/contrib/pyros/util.py | 71 +++++++++++++------ 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/pyomo/contrib/pyros/tests/test_preprocessor.py b/pyomo/contrib/pyros/tests/test_preprocessor.py index e1aed527e9c..50377773bf5 100644 --- a/pyomo/contrib/pyros/tests/test_preprocessor.py +++ b/pyomo/contrib/pyros/tests/test_preprocessor.py @@ -2795,8 +2795,6 @@ def test_preprocessor_objective_standardization(self, name, dr_order): + ublk.p**3 * (ublk.z1 + ublk.z2 + ublk.y1) + ublk.z4 + ublk.z5 - # TODO: are we sure this shouldn't be q2expr.expr? - # expected removal of named expression + ublk.q2expr ), ) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 16edf4cd935..4be4e09cd68 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -1661,14 +1661,53 @@ def turn_adjustable_var_bounds_to_constraints(model_data): # the interface for separation priority ordering +def _replace_vars_in_component_exprs(block, substitution_map, ctype): + """ + Substitute other objects for Vars in the expression attributes + of the component objects of a given type in a given block. + + For efficiency purposes, only components whose expressions + contain the Vars to remove via the substitution are acted upon. + + Named expressions in the components acted upon are descended + into, but not removed. + + Parameters + ---------- + block : BlockData + Block on which to perform the replacement. + substitution_map : ComponentMap + First entry of each tuple is a Var to remove, + second entry is an object to introduce in its place. + ctype : type or tuple of type + Type(s) of the components whose expressions are to be + modified. + """ + vars_to_be_replaced = ComponentSet([var for var, _ in substitution_map.items()]) + substitution_map = {id(var): dest for var, dest in substitution_map.items()} + for cdata in block.component_data_objects(ctype, active=None, descend_into=True): + # efficiency: act only on components containing + # the Vars to be substituted + if ComponentSet(identify_variables(cdata.expr)) & vars_to_be_replaced: + cdata.set_value( + replace_expressions( + expr=cdata.expr, + substitution_map=substitution_map, + descend_into_named_expressions=True, + remove_named_expressions=False, + ) + ) + + def replace_vars_with_params(block, var_to_param_map): """ Substitute ParamData objects for VarData objects - in all named expression, constraint, and objective components + in the Expression, Constraint, and Objective components declared on a block and all its sub-blocks. - Named Expressions are removed from Constraint and Objective - components during the substitution. + Note that when performing the substitutions in the + Constraint and Objective components, + named Expressions are descended into, but not replaced. Parameters ---------- @@ -1678,24 +1717,14 @@ def replace_vars_with_params(block, var_to_param_map): Mapping from VarData objects to be replaced to the ParamData objects to be introduced. """ - substitution_map = {id(var): param for var, param in var_to_param_map.items()} - vars_to_replace = ComponentSet(var_to_param_map.keys()) - ctypes = (Constraint, Objective, Expression) - cdata_objs = ( - cdata - for cdata in block.component_data_objects( - ctype=ctypes, active=None, descend_into=True - ) - if ComponentSet(identify_variables(cdata.expr)) & vars_to_replace - ) - for cdata in cdata_objs: - cdata.set_value( - replace_expressions( - expr=cdata.expr, - substitution_map=substitution_map, - descend_into_named_expressions=True, - remove_named_expressions=True, - ) + # invoke the expression replacement method for each + # individual component type to ensure the substitutions + # are always performed in the named expressions first + for ctype in (Expression, Constraint, Objective): + _replace_vars_in_component_exprs( + block=block, + substitution_map=var_to_param_map, + ctype=ctype, ) From 972ae6c3a21af86a11b4f6374a3bf29276669d3c Mon Sep 17 00:00:00 2001 From: jasherma Date: Sat, 30 Nov 2024 16:55:12 -0500 Subject: [PATCH 08/36] Modify uncertain parameter replacement logging messages --- pyomo/contrib/pyros/util.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 4be4e09cd68..75acb3523f4 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -40,6 +40,7 @@ maximize, minimize, Param, + ParamData, Reals, Var, VarData, @@ -1787,15 +1788,6 @@ def setup_working_model(model_data, user_var_partitioning): uncertain_param_var_idxs = [] for idx, obj in enumerate(working_model.orig_uncertain_params): if isinstance(obj, VarData): - config.progress_logger.debug( - "Entry of argument `uncertain_params` with name " - f"{obj.name!r} is of type {VarData.__name__}. " - f"Bounds and fixing for this {VarData.__name__} object " - "will be ignored. " - "A temporary ParamData object will be substituted for " - f"{obj.name!r} in all expressions, constraints, and objectives " - "of the working model clone. " - ) obj.fix() uncertain_param_var_idxs.append(idx) temp_params = working_model.temp_uncertain_params = Param( @@ -1815,13 +1807,27 @@ def setup_working_model(model_data, user_var_partitioning): # don't want to pass over the model components unless # at least one Var is to be replaced if uncertain_param_var_idxs: + uncertain_var_to_param_map = ComponentMap( + (working_model.orig_uncertain_params[idx], temp_param) + for idx, temp_param in temp_params.items() + ) replace_vars_with_params( working_model, - var_to_param_map=ComponentMap( - (working_model.orig_uncertain_params[idx], temp_param) - for idx, temp_param in temp_params.items() - ), + var_to_param_map=uncertain_var_to_param_map, ) + for var, param in uncertain_var_to_param_map.items(): + config.progress_logger.debug( + "Uncertain parameter with name " + f"{var.name!r} (relative to the working model clone) " + f"is of type {VarData.__name__}. " + f"The user-specified domain, declared bounds, and fixing of " + f"this {VarData.__name__} object will be ignored. " + f"A newly declared {ParamData.__name__} object " + f"with name {param.name!r} " + f"has been substituted for the {VarData.__name__} object " + "in all named expressions, constraints, and objectives " + "of the working model clone. " + ) # keep track of the original active constraints working_model.original_active_equality_cons = [] From c7cf91ac872c767b14f2339efb3d583cc057f1ff Mon Sep 17 00:00:00 2001 From: jasherma Date: Sat, 30 Nov 2024 17:08:20 -0500 Subject: [PATCH 09/36] Note Var bounds/domains/fixing ignored in PyROS docstring --- pyomo/contrib/pyros/pyros.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyomo/contrib/pyros/pyros.py b/pyomo/contrib/pyros/pyros.py index 18728acc636..e4bc6e6a7e9 100644 --- a/pyomo/contrib/pyros/pyros.py +++ b/pyomo/contrib/pyros/pyros.py @@ -303,6 +303,8 @@ def solve( Uncertain model parameters. Of every constituent `Param` and `ParamData` object, the `mutable` attribute must be set to True. + Of every constituent `Var` and `VarData` object, + the domain, declared bounds, and fixing are ignored. uncertainty_set: UncertaintySet Uncertainty set against which the solution(s) returned will be confirmed to be robust. From 05dfbb6c28db4ce6c990eb617ab3f3fe9d373db4 Mon Sep 17 00:00:00 2001 From: jasherma Date: Sat, 30 Nov 2024 17:12:38 -0500 Subject: [PATCH 10/36] Modify test for `InputDataStandardizer.domain_name()` --- pyomo/contrib/pyros/tests/test_config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/pyros/tests/test_config.py b/pyomo/contrib/pyros/tests/test_config.py index 5fc90c26d55..79e4679e91c 100644 --- a/pyomo/contrib/pyros/tests/test_config.py +++ b/pyomo/contrib/pyros/tests/test_config.py @@ -341,11 +341,16 @@ def test_standardizer_domain_name(self): Test domain name function works as expected. """ std1 = InputDataStandardizer(ctype=Param, cdatatype=ParamData) - self.assertEqual(std1.domain_name(), "(iterable of) Param, ParamData") + self.assertEqual( + std1.domain_name(), + f"(iterable of) {Param.__name__}, {ParamData.__name__}", + ) std2 = InputDataStandardizer(ctype=(Param, Var), cdatatype=(ParamData, VarData)) self.assertEqual( - std2.domain_name(), "(iterable of) Param, Var, ParamData, VarData" + std2.domain_name(), + f"(iterable of) {Param.__name__}, {Var.__name__}, " + f"{ParamData.__name__}, {VarData.__name__}" ) From 13e6ac589277e5722acf7c2a0ab0da6b26cb3c28 Mon Sep 17 00:00:00 2001 From: jasherma Date: Sat, 30 Nov 2024 17:13:18 -0500 Subject: [PATCH 11/36] Apply black --- pyomo/contrib/pyros/tests/test_config.py | 5 ++--- pyomo/contrib/pyros/tests/test_preprocessor.py | 10 ++-------- pyomo/contrib/pyros/util.py | 7 ++----- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/pyomo/contrib/pyros/tests/test_config.py b/pyomo/contrib/pyros/tests/test_config.py index 79e4679e91c..5d408dde79f 100644 --- a/pyomo/contrib/pyros/tests/test_config.py +++ b/pyomo/contrib/pyros/tests/test_config.py @@ -342,15 +342,14 @@ def test_standardizer_domain_name(self): """ std1 = InputDataStandardizer(ctype=Param, cdatatype=ParamData) self.assertEqual( - std1.domain_name(), - f"(iterable of) {Param.__name__}, {ParamData.__name__}", + std1.domain_name(), f"(iterable of) {Param.__name__}, {ParamData.__name__}" ) std2 = InputDataStandardizer(ctype=(Param, Var), cdatatype=(ParamData, VarData)) self.assertEqual( std2.domain_name(), f"(iterable of) {Param.__name__}, {Var.__name__}, " - f"{ParamData.__name__}, {VarData.__name__}" + f"{ParamData.__name__}, {VarData.__name__}", ) diff --git a/pyomo/contrib/pyros/tests/test_preprocessor.py b/pyomo/contrib/pyros/tests/test_preprocessor.py index 50377773bf5..4a1177a229f 100644 --- a/pyomo/contrib/pyros/tests/test_preprocessor.py +++ b/pyomo/contrib/pyros/tests/test_preprocessor.py @@ -36,13 +36,7 @@ Block, ) from pyomo.core.base.set_types import NonNegativeReals, NonPositiveReals, Reals -from pyomo.core.expr import ( - LinearExpression, - log, - sin, - exp, - RangedExpression, -) +from pyomo.core.expr import LinearExpression, log, sin, exp, RangedExpression from pyomo.core.expr.compare import assertExpressionsEqual from pyomo.contrib.pyros.util import ( @@ -2602,7 +2596,7 @@ def test_preprocessor_constraint_partitioning_nonstatic_dr( assertExpressionsEqual( self, ss.inequality_cons["ineq_con_ineq6_lower_bound_con"].expr, - -m.x1 <= - (-1 * working_model.temp_uncertain_params[1]), + -m.x1 <= -(-1 * working_model.temp_uncertain_params[1]), ) assertExpressionsEqual( diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 75acb3523f4..9b6f7d0250e 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -1723,9 +1723,7 @@ def replace_vars_with_params(block, var_to_param_map): # are always performed in the named expressions first for ctype in (Expression, Constraint, Objective): _replace_vars_in_component_exprs( - block=block, - substitution_map=var_to_param_map, - ctype=ctype, + block=block, substitution_map=var_to_param_map, ctype=ctype ) @@ -1812,8 +1810,7 @@ def setup_working_model(model_data, user_var_partitioning): for idx, temp_param in temp_params.items() ) replace_vars_with_params( - working_model, - var_to_param_map=uncertain_var_to_param_map, + working_model, var_to_param_map=uncertain_var_to_param_map ) for var, param in uncertain_var_to_param_map.items(): config.progress_logger.debug( From 3b4c59d8b4517ac84a610411b982745237dbd63c Mon Sep 17 00:00:00 2001 From: jasherma Date: Sat, 30 Nov 2024 23:06:43 -0500 Subject: [PATCH 12/36] Tweak documentation of PyROS `uncertain_params` argument --- pyomo/contrib/pyros/config.py | 6 ++++-- pyomo/contrib/pyros/pyros.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/pyros/config.py b/pyomo/contrib/pyros/config.py index 90a51087da7..098a1812870 100644 --- a/pyomo/contrib/pyros/config.py +++ b/pyomo/contrib/pyros/config.py @@ -516,8 +516,10 @@ def pyros_config(): description=( """ Uncertain model parameters. - The `mutable` attribute for all uncertain parameter - objects should be set to True. + Of every constituent `Param` object, + the `mutable` attribute must be set to True. + Of every constituent `Var` and `VarData` object, + the domain, declared bounds, and fixing are ignored. """ ), visibility=1, diff --git a/pyomo/contrib/pyros/pyros.py b/pyomo/contrib/pyros/pyros.py index e4bc6e6a7e9..37fa77a73dc 100644 --- a/pyomo/contrib/pyros/pyros.py +++ b/pyomo/contrib/pyros/pyros.py @@ -301,7 +301,7 @@ def solve( Second-stage model variables (or control variables). uncertain_params: (iterable of) Param, Var, ParamData, or VarData Uncertain model parameters. - Of every constituent `Param` and `ParamData` object, + Of every constituent `Param` object, the `mutable` attribute must be set to True. Of every constituent `Var` and `VarData` object, the domain, declared bounds, and fixing are ignored. From ec086dd22a3b5952b2e7e6fa27f7439cec7d1309 Mon Sep 17 00:00:00 2001 From: jasherma Date: Sun, 1 Dec 2024 16:50:54 -0500 Subject: [PATCH 13/36] Make some coeff matching tests more rigorous --- pyomo/contrib/pyros/tests/test_grcs.py | 44 ++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index 4383ed45602..4599ce6b19e 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -1664,6 +1664,50 @@ def test_coefficient_matching_robust_infeasible_proof_in_pyros(self): pyrosTerminationCondition.robust_infeasible, msg="Robust infeasible problem not identified via coefficient matching.", ) + self.assertEqual( + results.iterations, + 0, + msg="Number of PyROS iterations not as expected.", + ) + + @unittest.skipUnless(ipopt_available, "IPOPT not available") + def test_coefficient_matching_robust_infeasible_param_only_con(self): + """ + Test robust infeasibility reported due to equality + constraint depending only on uncertain params. + """ + m = build_leyffer() + m.robust_infeasible_eq_con = Constraint(expr=m.u == 1) + + box_set = BoxSet(bounds=[(0.25, 2)]) + + ipopt = SolverFactory("ipopt") + pyros_solver = SolverFactory("pyros") + + results = pyros_solver.solve( + model=m, + first_stage_variables=[m.x1, m.x2], + second_stage_variables=[], + uncertain_params=[m.u], + uncertainty_set=box_set, + local_solver=ipopt, + global_solver=ipopt, + options={ + "objective_focus": ObjectiveType.worst_case, + "solve_master_globally": True, + }, + ) + + self.assertEqual( + results.pyros_termination_condition, + pyrosTerminationCondition.robust_infeasible, + msg="Robust infeasible problem not identified via coefficient matching.", + ) + self.assertEqual( + results.iterations, + 0, + msg="Number of PyROS iterations not as expected.", + ) @unittest.skipUnless(ipopt_available, "IPOPT not available.") def test_coefficient_matching_nonlinear_expr(self): From d9588f18218a0945636cb2f770bfc7d1c6396e98 Mon Sep 17 00:00:00 2001 From: jasherma Date: Sun, 1 Dec 2024 16:53:17 -0500 Subject: [PATCH 14/36] Apply black --- pyomo/contrib/pyros/tests/test_grcs.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index 4599ce6b19e..e571ada85f2 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -1665,9 +1665,7 @@ def test_coefficient_matching_robust_infeasible_proof_in_pyros(self): msg="Robust infeasible problem not identified via coefficient matching.", ) self.assertEqual( - results.iterations, - 0, - msg="Number of PyROS iterations not as expected.", + results.iterations, 0, msg="Number of PyROS iterations not as expected." ) @unittest.skipUnless(ipopt_available, "IPOPT not available") @@ -1704,9 +1702,7 @@ def test_coefficient_matching_robust_infeasible_param_only_con(self): msg="Robust infeasible problem not identified via coefficient matching.", ) self.assertEqual( - results.iterations, - 0, - msg="Number of PyROS iterations not as expected.", + results.iterations, 0, msg="Number of PyROS iterations not as expected." ) @unittest.skipUnless(ipopt_available, "IPOPT not available.") From 2243d2ea73ad4926a5ca3228bd4c39850f3ad5ec Mon Sep 17 00:00:00 2001 From: jasherma Date: Sun, 1 Dec 2024 17:32:44 -0500 Subject: [PATCH 15/36] Tweak new var as uncertain param test --- pyomo/contrib/pyros/tests/test_grcs.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index e571ada85f2..6935dc2e7fd 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -1760,9 +1760,13 @@ def test_pyros_vars_as_uncertain_params(self): """ mdl1 = build_leyffer() - # clone: use a Var to represent the uncertain parameter + # clone: use a Var to represent the uncertain parameter. + # to ensure Var is out of scope of all subproblems + # as viewed by the subsolvers, + # let's make the bounds exclude the nominal value; + # PyROS should ignore these bounds as well mdl2 = mdl1.clone() - mdl2.uvar = Var(initialize=mdl2.u.value) + mdl2.uvar = Var(initialize=mdl2.u.value, bounds=(-1, 0)) mdl2.con.set_value( replace_expressions( expr=mdl2.con.expr, substitution_map={id(mdl2.u): mdl2.uvar} From 35e6487fc5382ac039dc20f6f94d6eef6ad62582 Mon Sep 17 00:00:00 2001 From: jasherma Date: Mon, 2 Dec 2024 15:28:12 -0500 Subject: [PATCH 16/36] Modify uncertain parameter refs in online docs --- doc/OnlineDocs/explanation/solvers/pyros.rst | 82 +++++++++++++------- 1 file changed, 53 insertions(+), 29 deletions(-) diff --git a/doc/OnlineDocs/explanation/solvers/pyros.rst b/doc/OnlineDocs/explanation/solvers/pyros.rst index ad3c99c1c11..6d8efa2c30a 100644 --- a/doc/OnlineDocs/explanation/solvers/pyros.rst +++ b/doc/OnlineDocs/explanation/solvers/pyros.rst @@ -329,23 +329,30 @@ The deterministic Pyomo model for *hydro* is shown below. .. note:: Primitive data (Python literals) that have been hard-coded within a - deterministic model cannot be later considered uncertain, - unless they are first converted to ``Param`` objects within - the ``ConcreteModel`` object. - Furthermore, any ``Param`` object that is to be later considered - uncertain must have the property ``mutable=True``. + deterministic model (:class:`~pyomo.core.base.PyomoModel.ConcreteModel`) + cannot be later considered uncertain, + unless they are first converted to Pyomo + :class:`~pyomo.core.base.param.Param` instances declared on the + :class:`~pyomo.core.base.PyomoModel.ConcreteModel` object. + Furthermore, any :class:`~pyomo.core.base.param.Param` + object that is to be later considered uncertain must be instantiated + with the argument ``mutable=True``. .. note:: - In case modifying the ``mutable`` property inside the deterministic - model object itself is not straightforward in your context, - you may consider adding the following statement **after** + If specifying/modifying the ``mutable`` argument in the + :class:`~pyomo.core.base.param.Param` declarations + of your deterministic model source code + is not straightforward in your context, then + you may consider adding **after** the line ``import pyomo.environ as pyo`` but **before** defining the model - object: ``pyo.Param.DefaultMutable = True``. - For all ``Param`` objects declared after this statement, - the attribute ``mutable`` is set to ``True`` by default. - Hence, non-mutable ``Param`` objects are now declared by - explicitly passing the argument ``mutable=False`` to the - ``Param`` constructor. + object the statement: ``pyo.Param.DefaultMutable = True``. + For all :class:`~pyomo.core.base.param.Param` + objects declared after this statement, + the attribute ``mutable`` is set to True by default. + Hence, non-mutable :class:`~pyomo.core.base.param.Param` + objects are now declared by explicitly passing the argument + ``mutable=False`` to the :class:`~pyomo.core.base.param.Param` + constructor. .. doctest:: @@ -428,22 +435,39 @@ The deterministic Pyomo model for *hydro* is shown below. Step 2: Define the Uncertainty ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -First, we need to collect into a list those ``Param`` objects of our model -that represent potentially uncertain parameters. -For the purposes of our example, we shall assume uncertainty in the model -parameters ``[m.p[0], m.p[1], m.p[2], m.p[3]]``, for which we can -conveniently utilize the object ``m.p`` (itself an indexed ``Param`` object). +We first collect the components of our model that represent the +uncertain parameters. +In this example, we assume uncertainty in +the parameter objects ``m.p[0]``, ``m.p[1]``, ``m.p[2]``, and ``m.p[3]``. +Since these objects comprise the mutable :class:`~pyomo.core.base.param.Param` +object ``m.p``, we can conveniently specify: .. doctest:: - >>> # === Specify which parameters are uncertain === - >>> # We can pass IndexedParams this way to PyROS, - >>> # or as an expanded list per index - >>> uncertain_parameters = [m.p] + >>> uncertain_params = m.p + +Equivalently, we may instead set ``uncertain_params`` to +either ``[m.p]``, ``[m.p[0], m.p[1], m.p[2], m.p[3]]``, or ``list(m.p.values())``. +The PyROS :meth:`~pyomo.contrib.pyros.pyros.PyROS.solve` method +will upon invocation cast any of these to the list +``[m.p[0], m.p[1], m.p[2], m.p[3]]``. + +.. note:: + Any :class:`~pyomo.core.base.param.Param` object that is + to be considered uncertain by PyROS must have the property + ``mutable=True``. .. note:: - Any ``Param`` object that is to be considered uncertain by PyROS - must have the property ``mutable=True``. + PyROS also supports uncertain parameters implemented as + :class:`~pyomo.core.base.var.Var` objects declared on the + deterministic model. + This may be convenient for users transitioning to PyROS from + uncertainty quantification workflows. + Note that for each :class:`~pyomo.core.base.var.Var` object + that represents an uncertain parameter, + the domain and bounds specified through the attributes of the + :class:`~pyomo.core.base.var.Var` are ignored. + PyROS will seek to identify solutions that remain feasible for any realization of these parameters included in an uncertainty set. @@ -555,7 +579,7 @@ correspond to first-stage degrees of freedom. ... model=m, ... first_stage_variables=first_stage_variables, ... second_stage_variables=second_stage_variables, - ... uncertain_params=uncertain_parameters, + ... uncertain_params=uncertain_params, ... uncertainty_set=box_uncertainty_set, ... local_solver=local_solver, ... global_solver=global_solver, @@ -648,7 +672,7 @@ In this example, we select affine decision rules by setting ... model=m, ... first_stage_variables=first_stage_variables, ... second_stage_variables=second_stage_variables, - ... uncertain_params=uncertain_parameters, + ... uncertain_params=uncertain_params, ... uncertainty_set=box_uncertainty_set, ... local_solver=local_solver, ... global_solver=global_solver, @@ -702,7 +726,7 @@ could have been equivalently written as: ... model=m, ... first_stage_variables=first_stage_variables, ... second_stage_variables=second_stage_variables, - ... uncertain_params=uncertain_parameters, + ... uncertain_params=uncertain_params, ... uncertainty_set=box_uncertainty_set, ... local_solver=local_solver, ... global_solver=global_solver, @@ -768,7 +792,7 @@ instance and invoking the PyROS solver: ... model=m, ... first_stage_variables=first_stage_variables, ... second_stage_variables=second_stage_variables, - ... uncertain_params=uncertain_parameters, + ... uncertain_params=uncertain_params, ... uncertainty_set= box_uncertainty_set, ... local_solver=local_solver, ... global_solver=global_solver, From c372d98e5990a953be5898b39e7b63473e68370a Mon Sep 17 00:00:00 2001 From: jasherma Date: Mon, 2 Dec 2024 15:32:11 -0500 Subject: [PATCH 17/36] Tweak note on vars as params --- doc/OnlineDocs/explanation/solvers/pyros.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/OnlineDocs/explanation/solvers/pyros.rst b/doc/OnlineDocs/explanation/solvers/pyros.rst index 6d8efa2c30a..0a6fdf49efa 100644 --- a/doc/OnlineDocs/explanation/solvers/pyros.rst +++ b/doc/OnlineDocs/explanation/solvers/pyros.rst @@ -465,8 +465,8 @@ will upon invocation cast any of these to the list uncertainty quantification workflows. Note that for each :class:`~pyomo.core.base.var.Var` object that represents an uncertain parameter, - the domain and bounds specified through the attributes of the - :class:`~pyomo.core.base.var.Var` are ignored. + PyROS ignores the domain and bounds specified + through the attributes of the :class:`~pyomo.core.base.var.Var`. PyROS will seek to identify solutions that remain feasible for any From 21e28a109285233367d0f1b790ba4d20688df6d5 Mon Sep 17 00:00:00 2001 From: jasherma Date: Fri, 20 Dec 2024 08:02:54 -0500 Subject: [PATCH 18/36] Update version number, changelog --- pyomo/contrib/pyros/CHANGELOG.txt | 7 +++++++ pyomo/contrib/pyros/pyros.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/pyros/CHANGELOG.txt b/pyomo/contrib/pyros/CHANGELOG.txt index 0036311fc91..a17fbf23641 100644 --- a/pyomo/contrib/pyros/CHANGELOG.txt +++ b/pyomo/contrib/pyros/CHANGELOG.txt @@ -2,6 +2,13 @@ PyROS CHANGELOG =============== +------------------------------------------------------------------------------- +PyROS 1.3.2 29 Nov 2024 +------------------------------------------------------------------------------- +- Allow Var/VarData objects to be specified as uncertain parameters + through the `uncertain_params` argument to `PyROS.solve()` + + ------------------------------------------------------------------------------- PyROS 1.3.1 25 Nov 2024 ------------------------------------------------------------------------------- diff --git a/pyomo/contrib/pyros/pyros.py b/pyomo/contrib/pyros/pyros.py index 5405857249a..57337863351 100644 --- a/pyomo/contrib/pyros/pyros.py +++ b/pyomo/contrib/pyros/pyros.py @@ -33,7 +33,7 @@ ) -__version__ = "1.3.1" +__version__ = "1.3.2" default_pyros_solver_logger = setup_pyros_logger() From 1bde454e42c67be10f6391d64de92253ef026cd0 Mon Sep 17 00:00:00 2001 From: jasherma Date: Fri, 20 Dec 2024 08:05:43 -0500 Subject: [PATCH 19/36] Add bullet on DEBUG-level output logs --- doc/OnlineDocs/explanation/solvers/pyros.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/OnlineDocs/explanation/solvers/pyros.rst b/doc/OnlineDocs/explanation/solvers/pyros.rst index a836db82c99..8f7e2548f8a 100644 --- a/doc/OnlineDocs/explanation/solvers/pyros.rst +++ b/doc/OnlineDocs/explanation/solvers/pyros.rst @@ -884,7 +884,8 @@ for a basic tutorial, see the :doc:`logging HOWTO `. * Iteration log table * Termination details: message, timing breakdown, summary of statistics * - :py:obj:`logging.DEBUG` - - * Termination outcomes and summary of statistics for + - * Progress through the various preprocessing subroutines + * Termination outcomes and summary of statistics for every master feasility, master, and DR polishing problem * Progress updates for the separation procedure * Separation subproblem initial point infeasibilities @@ -959,7 +960,7 @@ Observe that the log contains the following information: :linenos: ============================================================================== - PyROS: The Pyomo Robust Optimization Solver, v1.3.1. + PyROS: The Pyomo Robust Optimization Solver, v1.3.2. Pyomo version: 6.9.0 Commit hash: unknown Invoked at UTC 2024-11-01T00:00:00.000000 From c3fc42809afbd1a878a594e3eb194cb92dc7cc3e Mon Sep 17 00:00:00 2001 From: jasherma Date: Sun, 19 Jan 2025 21:21:36 -0500 Subject: [PATCH 20/36] Require that `Vars` passed as uncertain params be fixed --- pyomo/contrib/pyros/config.py | 35 ++++++- pyomo/contrib/pyros/pyros.py | 4 +- pyomo/contrib/pyros/tests/test_config.py | 58 +++++++++++ pyomo/contrib/pyros/tests/test_grcs.py | 122 ++++++++++++++++++----- pyomo/contrib/pyros/util.py | 42 +++++++- 5 files changed, 233 insertions(+), 28 deletions(-) diff --git a/pyomo/contrib/pyros/config.py b/pyomo/contrib/pyros/config.py index 098a1812870..54446c357e9 100644 --- a/pyomo/contrib/pyros/config.py +++ b/pyomo/contrib/pyros/config.py @@ -85,13 +85,45 @@ def uncertain_param_validator(uncertain_obj): "Check that the component has been properly constructed, " "and all entries have been initialized. " ) - if isinstance(uncertain_obj, (Param, ParamData)) and not uncertain_obj.mutable: + if isinstance(uncertain_obj, Param) and not uncertain_obj.mutable: raise ValueError( f"{type(uncertain_obj).__name__} object with name {uncertain_obj.name!r} " "is immutable." ) +def uncertain_param_data_validator(uncertain_obj): + """ + Validator for component data object specified as an + uncertain parameter. + + Parameters + ---------- + uncertain_obj : Param or Var + Object on which to perform checks. + + Raises + ------ + ValueError + If `uncertain_obj` is a VarData object + that is not fixed explicitly via VarData.fixed + or implicitly via bounds. + """ + if isinstance(uncertain_obj, VarData): + is_fixed_var = ( + uncertain_obj.fixed + or ( + uncertain_obj.lower is uncertain_obj.upper + and uncertain_obj.lower is not None + ) + ) + if not is_fixed_var: + raise ValueError( + f"{type(uncertain_obj).__name__} object with name " + f"{uncertain_obj.name!r} is not fixed." + ) + + class InputDataStandardizer(object): """ Domain validator for an object that is castable to @@ -511,6 +543,7 @@ def pyros_config(): ctype=(Param, Var), cdatatype=(ParamData, VarData), ctype_validator=uncertain_param_validator, + cdatatype_validator=uncertain_param_data_validator, allow_repeats=False, ), description=( diff --git a/pyomo/contrib/pyros/pyros.py b/pyomo/contrib/pyros/pyros.py index 57337863351..3044658cb39 100644 --- a/pyomo/contrib/pyros/pyros.py +++ b/pyomo/contrib/pyros/pyros.py @@ -303,8 +303,8 @@ def solve( Uncertain model parameters. Of every constituent `Param` object, the `mutable` attribute must be set to True. - Of every constituent `Var` and `VarData` object, - the domain, declared bounds, and fixing are ignored. + All constituent `Var`/`VarData` objects should be + fixed. uncertainty_set: UncertaintySet Uncertainty set against which the solution(s) returned will be confirmed to be robust. diff --git a/pyomo/contrib/pyros/tests/test_config.py b/pyomo/contrib/pyros/tests/test_config.py index 5d408dde79f..a9962326580 100644 --- a/pyomo/contrib/pyros/tests/test_config.py +++ b/pyomo/contrib/pyros/tests/test_config.py @@ -23,6 +23,7 @@ from pyomo.contrib.pyros.config import ( InputDataStandardizer, uncertain_param_validator, + uncertain_param_data_validator, logger_domain, SolverNotResolvable, positive_int_or_minus_one, @@ -336,6 +337,63 @@ def test_standardizer_multiple_ctypes_with_validator(self): ), ) + def test_standardizer_with_both_validators(self): + """ + Test input data standardizer when there is + a validator for the component and component data types. + """ + mdl = ConcreteModel() + mdl.p = Param([0, 1], initialize=0, mutable=True) + mdl.v = Var(["a", "b"], initialize=1) + + standardizer_func = InputDataStandardizer( + ctype=(Var, Param), + cdatatype=(VarData, ParamData), + ctype_validator=uncertain_param_validator, + cdatatype_validator=uncertain_param_data_validator, + ) + + err_str_a = r".*VarData object with name 'v\[a\]' is not fixed" + err_str_b = r".*VarData object with name 'v\[b\]' is not fixed" + + with self.assertRaisesRegex(ValueError, err_str_a): + standardizer_func([mdl.p, mdl.v]) + + with self.assertRaisesRegex(ValueError, err_str_a): + standardizer_func([mdl.p, mdl.v["a"], mdl.v["b"]]) + + with self.assertRaisesRegex(ValueError, err_str_a): + standardizer_func(mdl.v["a"]) + + mdl.v["a"].fix() + va_output = standardizer_func(mdl.v["a"]) + self.assertEqual(va_output, [mdl.v["a"]]) + + with self.assertRaisesRegex(ValueError, err_str_b): + standardizer_func([mdl.p, mdl.v["a"], mdl.v["b"]]) + + mdl.v["b"].fix() + va_vb_output = standardizer_func([mdl.v["a"], mdl.v["b"]]) + self.assertEqual( + va_vb_output, + [mdl.v["a"], mdl.v["b"]], + ) + + va_vb_unraveled_output = standardizer_func(mdl.v) + self.assertEqual( + va_vb_unraveled_output, + [mdl.v["a"], mdl.v["b"]], + ) + + mdl.v["a"].unfix() + mdl.v["a"].setlb(1) + mdl.v["a"].setub(1) + va_vb_unraveled_output_2 = standardizer_func(mdl.v) + self.assertEqual( + va_vb_unraveled_output_2, + [mdl.v["a"], mdl.v["b"]], + ) + def test_standardizer_domain_name(self): """ Test domain name function works as expected. diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index 6935dc2e7fd..2d45d1fb04d 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -1752,12 +1752,14 @@ def test_coefficient_matching_nonlinear_expr(self): pyrosTerminationCondition.robust_feasible, ) - @unittest.skipUnless(ipopt_available, "IPOPT not available.") - def test_pyros_vars_as_uncertain_params(self): - """ - Test PyROS solver result is invariant to the type used - in argument `uncertain_params`. - """ + +@unittest.skipUnless(ipopt_available, "IPOPT not available.") +class TestPyROSVarsAsUncertainParams(unittest.TestCase): + """ + Test PyROS solver treatment of Var/VarData + objects passed as uncertain parameters. + """ + def build_model_objects(self): mdl1 = build_leyffer() # clone: use a Var to represent the uncertain parameter. @@ -1766,14 +1768,64 @@ def test_pyros_vars_as_uncertain_params(self): # let's make the bounds exclude the nominal value; # PyROS should ignore these bounds as well mdl2 = mdl1.clone() - mdl2.uvar = Var(initialize=mdl2.u.value, bounds=(-1, 0)) + mdl2.uvar = Var([0], initialize={0: mdl2.u.value}, bounds=(-1, 0)) + mdl2.uvar.fix() mdl2.con.set_value( replace_expressions( - expr=mdl2.con.expr, substitution_map={id(mdl2.u): mdl2.uvar} + expr=mdl2.con.expr, substitution_map={id(mdl2.u): mdl2.uvar[0]} ) ) - box_set = BoxSet([[0.25, 2]]) + + return mdl1, mdl2, box_set + + def test_pyros_unfixed_vars_as_uncertain_params(self): + """ + Test PyROS raises exception if unfixed Vars are + passed to the argument `uncertain_params`. + """ + _, mdl2, box_set = self.build_model_objects() + mdl2.uvar.unfix() + + ipopt_solver = SolverFactory("ipopt") + pyros_solver = SolverFactory("pyros") + + err_str = r".*VarData object with name 'uvar\[0\]' is not fixed" + + with self.assertRaisesRegex(ValueError, err_str): + pyros_solver.solve( + model=mdl2, + first_stage_variables=[mdl2.x1, mdl2.x2], + second_stage_variables=[], + uncertain_params=mdl2.uvar, + uncertainty_set=box_set, + local_solver=ipopt_solver, + global_solver=ipopt_solver, + ) + + with self.assertRaisesRegex(ValueError, err_str): + pyros_solver.solve( + model=mdl2, + first_stage_variables=[mdl2.x1, mdl2.x2], + second_stage_variables=[], + uncertain_params=[mdl2.uvar[0]], + uncertainty_set=box_set, + local_solver=ipopt_solver, + global_solver=ipopt_solver, + ) + + def test_pyros_vars_as_uncertain_params_correct(self): + """ + Test PyROS solver result is invariant to the type used + in argument `uncertain_params`. + """ + mdl1, mdl2, box_set = self.build_model_objects() + mdl3 = mdl2.clone() + + mdl3.uvar.unfix() + mdl3.uvar[0].setlb(mdl3.uvar[0].value) + mdl3.uvar[0].setub(mdl3.uvar[0].value) + ipopt_solver = SolverFactory("ipopt") pyros_solver = SolverFactory("pyros") @@ -1790,20 +1842,43 @@ def test_pyros_vars_as_uncertain_params(self): res1.pyros_termination_condition, pyrosTerminationCondition.robust_feasible ) - res2 = pyros_solver.solve( - model=mdl2, - first_stage_variables=[mdl2.x1, mdl2.x2], - second_stage_variables=[], - uncertain_params=[mdl2.uvar], - uncertainty_set=box_set, - local_solver=ipopt_solver, - global_solver=ipopt_solver, - ) - self.assertEqual( - res2.pyros_termination_condition, res1.pyros_termination_condition - ) - self.assertEqual(res1.final_objective_value, res2.final_objective_value) - self.assertEqual(res1.iterations, res2.iterations) + for model, adverb in zip([mdl2, mdl3], ["explicitly", "by bounds"]): + res = pyros_solver.solve( + model=model, + first_stage_variables=[model.x1, model.x2], + second_stage_variables=[], + uncertain_params=model.uvar, + uncertainty_set=box_set, + local_solver=ipopt_solver, + global_solver=ipopt_solver, + ) + self.assertEqual( + res.pyros_termination_condition, + res1.pyros_termination_condition, + msg=( + "PyROS termination condition " + "is sensitive to uncertain parameter component type " + f"when uncertain parameter is a Var fixed {adverb}." + ), + ) + self.assertEqual( + res1.final_objective_value, + res.final_objective_value, + msg=( + "PyROS termination condition " + "is sensitive to uncertain parameter component type " + f"when uncertain parameter is a Var fixed {adverb}." + ), + ) + self.assertEqual( + res1.iterations, + res.iterations, + msg=( + "PyROS iteration count " + "is sensitive to uncertain parameter component type " + f"when uncertain parameter is a Var fixed {adverb}." + ), + ) @unittest.skipUnless(scip_available, "Global NLP solver is not available.") @@ -3045,6 +3120,7 @@ def test_pyros_overlap_uncertain_params_vars(self, stage_name, is_first_stage): "contain at least one common Var object." ) with LoggingIntercept(level=logging.ERROR) as LOG: + mdl.x1.fix() # uncertain params should be fixed with self.assertRaisesRegex(ValueError, exc_str): pyros.solve( model=mdl, diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 9b6f7d0250e..d52aeddc731 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -603,7 +603,11 @@ def standardize_component_data( if isinstance(obj, valid_ctype): if ctype_validator is not None: ctype_validator(obj) - return list(obj.values()) + ans = list(obj.values()) + if cdatatype_validator is not None: + for entry in ans: + cdatatype_validator(entry) + return ans elif isinstance(obj, valid_cdatatype): if cdatatype_validator is not None: cdatatype_validator(obj) @@ -861,6 +865,40 @@ def validate_variable_partitioning(model, config): ) +def _get_uncertain_param_val(var_or_param_data): + """ + Get value of VarData/ParamData object + that is considered an uncertain parameter. + + Parameters + ---------- + var_or_param_data : VarData or ParamData + Object to be evaluated. + + Returns + ------- + object + Value of the VarData/ParamData object. + The value is typically of a numeric type. + """ + if isinstance(var_or_param_data, ParamData): + expr_to_evaluate = var_or_param_data + elif isinstance(var_or_param_data, VarData): + if var_or_param_data.fixed: + expr_to_evaluate = var_or_param_data + else: + expr_to_evaluate = var_or_param_data.lower + else: + raise ValueError( + f"Uncertain parameter object {var_or_param_data!r}" + f"is of type {type(var_or_param_data).__name__!r}, " + "but should be of type " + f"{ParamData.__name__} or {VarData.__name__}." + ) + + return value(expr_to_evaluate, exception=True) + + def validate_uncertainty_specification(model, config): """ Validate specification of uncertain parameters and uncertainty @@ -932,7 +970,7 @@ def validate_uncertainty_specification(model, config): # otherwise, check length matches uncertainty dimension if not config.nominal_uncertain_param_vals: config.nominal_uncertain_param_vals = [ - value(param, exception=True) for param in config.uncertain_params + _get_uncertain_param_val(param) for param in config.uncertain_params ] elif len(config.nominal_uncertain_param_vals) != len(config.uncertain_params): raise ValueError( From 04eb8aa850778f8b6a33572e9332ccde18d39b23 Mon Sep 17 00:00:00 2001 From: jasherma Date: Sun, 19 Jan 2025 21:22:11 -0500 Subject: [PATCH 21/36] Apply black --- pyomo/contrib/pyros/tests/test_grcs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index 2d45d1fb04d..8901b154422 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -1759,6 +1759,7 @@ class TestPyROSVarsAsUncertainParams(unittest.TestCase): Test PyROS solver treatment of Var/VarData objects passed as uncertain parameters. """ + def build_model_objects(self): mdl1 = build_leyffer() From df3ee821b7e0fb50407ab3f917282a64c0b77171 Mon Sep 17 00:00:00 2001 From: jasherma Date: Sun, 19 Jan 2025 21:24:26 -0500 Subject: [PATCH 22/36] Complete black reformatting --- pyomo/contrib/pyros/config.py | 9 +++------ pyomo/contrib/pyros/tests/test_config.py | 15 +++------------ 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/pyomo/contrib/pyros/config.py b/pyomo/contrib/pyros/config.py index 54446c357e9..e3356ebc99f 100644 --- a/pyomo/contrib/pyros/config.py +++ b/pyomo/contrib/pyros/config.py @@ -110,12 +110,9 @@ def uncertain_param_data_validator(uncertain_obj): or implicitly via bounds. """ if isinstance(uncertain_obj, VarData): - is_fixed_var = ( - uncertain_obj.fixed - or ( - uncertain_obj.lower is uncertain_obj.upper - and uncertain_obj.lower is not None - ) + is_fixed_var = uncertain_obj.fixed or ( + uncertain_obj.lower is uncertain_obj.upper + and uncertain_obj.lower is not None ) if not is_fixed_var: raise ValueError( diff --git a/pyomo/contrib/pyros/tests/test_config.py b/pyomo/contrib/pyros/tests/test_config.py index a9962326580..5d4b12ddcb5 100644 --- a/pyomo/contrib/pyros/tests/test_config.py +++ b/pyomo/contrib/pyros/tests/test_config.py @@ -374,25 +374,16 @@ def test_standardizer_with_both_validators(self): mdl.v["b"].fix() va_vb_output = standardizer_func([mdl.v["a"], mdl.v["b"]]) - self.assertEqual( - va_vb_output, - [mdl.v["a"], mdl.v["b"]], - ) + self.assertEqual(va_vb_output, [mdl.v["a"], mdl.v["b"]]) va_vb_unraveled_output = standardizer_func(mdl.v) - self.assertEqual( - va_vb_unraveled_output, - [mdl.v["a"], mdl.v["b"]], - ) + self.assertEqual(va_vb_unraveled_output, [mdl.v["a"], mdl.v["b"]]) mdl.v["a"].unfix() mdl.v["a"].setlb(1) mdl.v["a"].setub(1) va_vb_unraveled_output_2 = standardizer_func(mdl.v) - self.assertEqual( - va_vb_unraveled_output_2, - [mdl.v["a"], mdl.v["b"]], - ) + self.assertEqual(va_vb_unraveled_output_2, [mdl.v["a"], mdl.v["b"]]) def test_standardizer_domain_name(self): """ From d2b98c17c38c3c3d8bb1c1de5d13c01dadeb765b Mon Sep 17 00:00:00 2001 From: jasherma Date: Mon, 20 Jan 2025 13:53:59 -0500 Subject: [PATCH 23/36] More rigorously test uncertain param data validator --- pyomo/contrib/pyros/tests/test_config.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pyomo/contrib/pyros/tests/test_config.py b/pyomo/contrib/pyros/tests/test_config.py index 5d4b12ddcb5..73bf9145b3a 100644 --- a/pyomo/contrib/pyros/tests/test_config.py +++ b/pyomo/contrib/pyros/tests/test_config.py @@ -379,12 +379,30 @@ def test_standardizer_with_both_validators(self): va_vb_unraveled_output = standardizer_func(mdl.v) self.assertEqual(va_vb_unraveled_output, [mdl.v["a"], mdl.v["b"]]) + # the param data validator supports unfixed Vars that + # have identical bounds mdl.v["a"].unfix() mdl.v["a"].setlb(1) mdl.v["a"].setub(1) va_vb_unraveled_output_2 = standardizer_func(mdl.v) self.assertEqual(va_vb_unraveled_output_2, [mdl.v["a"], mdl.v["b"]]) + # ensure exception raised if the bounds are not identical + # (even if equal in value) + mdl.v["a"].setlb(1.0) + with self.assertRaisesRegex(ValueError, err_str_a): + standardizer_func([mdl.p, mdl.v["a"], mdl.v["b"]]) + + mdl.q = Param(initialize=1, mutable=True) + mdl.v["a"].setlb(mdl.q) + with self.assertRaisesRegex(ValueError, err_str_a): + standardizer_func([mdl.p, mdl.v["a"], mdl.v["b"]]) + + # support fixing by bounds that are identical mutable expressions + mdl.v["a"].setub(mdl.q) + va_vb_unraveled_output_2 = standardizer_func(mdl.v) + self.assertEqual(va_vb_unraveled_output_2, [mdl.v["a"], mdl.v["b"]]) + def test_standardizer_domain_name(self): """ Test domain name function works as expected. From e55d0c537cbafad6fcbf30ff7b48bf40fcbc546f Mon Sep 17 00:00:00 2001 From: jasherma Date: Mon, 20 Jan 2025 13:54:39 -0500 Subject: [PATCH 24/36] Add note on uncertain params fixed to other uncertain params --- pyomo/contrib/pyros/util.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index d52aeddc731..6b46f034380 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -970,6 +970,12 @@ def validate_uncertainty_specification(model, config): # otherwise, check length matches uncertainty dimension if not config.nominal_uncertain_param_vals: config.nominal_uncertain_param_vals = [ + # NOTE: this allows uncertain parameters that are of type + # VarData and implicitly fixed by identical bounds + # that are mutable expressions in ParamData-type + # uncertain parameters; + # the bounds expressions are evaluated to + # to get the nominal realization _get_uncertain_param_val(param) for param in config.uncertain_params ] elif len(config.nominal_uncertain_param_vals) != len(config.uncertain_params): From 766311123aad3d08d0cf1782632fb9168483ee54 Mon Sep 17 00:00:00 2001 From: jasherma Date: Mon, 20 Jan 2025 14:44:02 -0500 Subject: [PATCH 25/36] Make PyROS uncertain params as vars tests more rigorous --- pyomo/contrib/pyros/tests/test_grcs.py | 74 ++++++++++++++++++++------ 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index 8901b154422..018798cb618 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -46,6 +46,7 @@ Block, ConcreteModel, Constraint, + Expression, Objective, Param, SolverFactory, @@ -1761,7 +1762,7 @@ class TestPyROSVarsAsUncertainParams(unittest.TestCase): """ def build_model_objects(self): - mdl1 = build_leyffer() + mdl1 = build_leyffer_two_cons_two_params() # clone: use a Var to represent the uncertain parameter. # to ensure Var is out of scope of all subproblems @@ -1769,14 +1770,26 @@ def build_model_objects(self): # let's make the bounds exclude the nominal value; # PyROS should ignore these bounds as well mdl2 = mdl1.clone() - mdl2.uvar = Var([0], initialize={0: mdl2.u.value}, bounds=(-1, 0)) - mdl2.uvar.fix() - mdl2.con.set_value( - replace_expressions( - expr=mdl2.con.expr, substitution_map={id(mdl2.u): mdl2.uvar[0]} + mdl2.uvar = Var( + [1, 2], initialize={1: mdl2.u1.value, 2: mdl2.u2.value}, bounds=(-1, 0) + ) + + # want to test replacement of named expressions + # in preprocessing as well, + # so we add a simple placeholder expression + mdl2.uvar2_expr = Expression(expr=mdl2.uvar[2]) + + for comp in [mdl2.con1, mdl2.con2, mdl2.obj]: + comp.set_value( + replace_expressions( + expr=comp.expr, + substitution_map={ + id(mdl2.u1): mdl2.uvar[1], + id(mdl2.u2): mdl2.uvar2_expr, + }, + ) ) - ) - box_set = BoxSet([[0.25, 2]]) + box_set = BoxSet([[0.25, 2], [0.5, 1.5]]) return mdl1, mdl2, box_set @@ -1791,9 +1804,8 @@ def test_pyros_unfixed_vars_as_uncertain_params(self): ipopt_solver = SolverFactory("ipopt") pyros_solver = SolverFactory("pyros") - err_str = r".*VarData object with name 'uvar\[0\]' is not fixed" - - with self.assertRaisesRegex(ValueError, err_str): + err_str_1 = r".*VarData object with name 'uvar\[1\]' is not fixed" + with self.assertRaisesRegex(ValueError, err_str_1): pyros_solver.solve( model=mdl2, first_stage_variables=[mdl2.x1, mdl2.x2], @@ -1803,13 +1815,35 @@ def test_pyros_unfixed_vars_as_uncertain_params(self): local_solver=ipopt_solver, global_solver=ipopt_solver, ) + with self.assertRaisesRegex(ValueError, err_str_1): + pyros_solver.solve( + model=mdl2, + first_stage_variables=[mdl2.x1, mdl2.x2], + second_stage_variables=[], + uncertain_params=[mdl2.uvar[1], mdl2.uvar[2]], + uncertainty_set=box_set, + local_solver=ipopt_solver, + global_solver=ipopt_solver, + ) - with self.assertRaisesRegex(ValueError, err_str): + mdl2.uvar[1].fix() + err_str_2 = r".*VarData object with name 'uvar\[2\]' is not fixed" + with self.assertRaisesRegex(ValueError, err_str_2): pyros_solver.solve( model=mdl2, first_stage_variables=[mdl2.x1, mdl2.x2], second_stage_variables=[], - uncertain_params=[mdl2.uvar[0]], + uncertain_params=mdl2.uvar, + uncertainty_set=box_set, + local_solver=ipopt_solver, + global_solver=ipopt_solver, + ) + with self.assertRaisesRegex(ValueError, err_str_2): + pyros_solver.solve( + model=mdl2, + first_stage_variables=[mdl2.x1, mdl2.x2], + second_stage_variables=[], + uncertain_params=[mdl2.uvar[1], mdl2.uvar[2]], uncertainty_set=box_set, local_solver=ipopt_solver, global_solver=ipopt_solver, @@ -1821,11 +1855,17 @@ def test_pyros_vars_as_uncertain_params_correct(self): in argument `uncertain_params`. """ mdl1, mdl2, box_set = self.build_model_objects() - mdl3 = mdl2.clone() + # explicitly fixed + mdl2.uvar.fix() + + # fixed by bounds that are literal constants + mdl3 = mdl2.clone() mdl3.uvar.unfix() - mdl3.uvar[0].setlb(mdl3.uvar[0].value) - mdl3.uvar[0].setub(mdl3.uvar[0].value) + mdl3.uvar[1].setlb(mdl3.uvar[1].value) + mdl3.uvar[1].setub(mdl3.uvar[1].value) + mdl3.uvar[2].setlb(mdl3.uvar[2].value) + mdl3.uvar[2].setub(mdl3.uvar[2].value) ipopt_solver = SolverFactory("ipopt") pyros_solver = SolverFactory("pyros") @@ -1834,7 +1874,7 @@ def test_pyros_vars_as_uncertain_params_correct(self): model=mdl1, first_stage_variables=[mdl1.x1, mdl1.x2], second_stage_variables=[], - uncertain_params=[mdl1.u], + uncertain_params=[mdl1.u1, mdl1.u2], uncertainty_set=box_set, local_solver=ipopt_solver, global_solver=ipopt_solver, From af1673699e7bdd4c125734ebb1b48447d86f401c Mon Sep 17 00:00:00 2001 From: jasherma Date: Mon, 20 Jan 2025 14:44:37 -0500 Subject: [PATCH 26/36] Apply black --- pyomo/contrib/pyros/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 6b46f034380..d4f2ec3f619 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -976,7 +976,8 @@ def validate_uncertainty_specification(model, config): # uncertain parameters; # the bounds expressions are evaluated to # to get the nominal realization - _get_uncertain_param_val(param) for param in config.uncertain_params + _get_uncertain_param_val(param) + for param in config.uncertain_params ] elif len(config.nominal_uncertain_param_vals) != len(config.uncertain_params): raise ValueError( From f85f10381b9874d185b0c60324434aae37586594 Mon Sep 17 00:00:00 2001 From: jasherma Date: Mon, 20 Jan 2025 14:57:42 -0500 Subject: [PATCH 27/36] Adjust PyROS online doc to new uncertain parameter restriction --- doc/OnlineDocs/explanation/solvers/pyros.rst | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/doc/OnlineDocs/explanation/solvers/pyros.rst b/doc/OnlineDocs/explanation/solvers/pyros.rst index 8f7e2548f8a..f27c901d96a 100644 --- a/doc/OnlineDocs/explanation/solvers/pyros.rst +++ b/doc/OnlineDocs/explanation/solvers/pyros.rst @@ -447,10 +447,8 @@ object ``m.p``, we can conveniently specify: >>> uncertain_params = m.p Equivalently, we may instead set ``uncertain_params`` to -either ``[m.p]``, ``[m.p[0], m.p[1], m.p[2], m.p[3]]``, or ``list(m.p.values())``. -The PyROS :meth:`~pyomo.contrib.pyros.pyros.PyROS.solve` method -will upon invocation cast any of these to the list -``[m.p[0], m.p[1], m.p[2], m.p[3]]``. +either ``[m.p]``, ``[m.p[0], m.p[1], m.p[2], m.p[3]]``, +or ``list(m.p.values())``. .. note:: Any :class:`~pyomo.core.base.param.Param` object that is @@ -463,10 +461,8 @@ will upon invocation cast any of these to the list deterministic model. This may be convenient for users transitioning to PyROS from uncertainty quantification workflows. - Note that for each :class:`~pyomo.core.base.var.Var` object - that represents an uncertain parameter, - PyROS ignores the domain and bounds specified - through the attributes of the :class:`~pyomo.core.base.var.Var`. + All :class:`~pyomo.core.base.var.Var` objects representing + uncertain parameters should be fixed. PyROS will seek to identify solutions that remain feasible for any From 17233f76c33c538e66165f827d8052df9443c2ef Mon Sep 17 00:00:00 2001 From: jasherma Date: Mon, 20 Jan 2025 15:26:14 -0500 Subject: [PATCH 28/36] Tweak doc note on support for `Var`s as uncertain parameters --- doc/OnlineDocs/explanation/solvers/pyros.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/OnlineDocs/explanation/solvers/pyros.rst b/doc/OnlineDocs/explanation/solvers/pyros.rst index f27c901d96a..f96b9285240 100644 --- a/doc/OnlineDocs/explanation/solvers/pyros.rst +++ b/doc/OnlineDocs/explanation/solvers/pyros.rst @@ -456,13 +456,15 @@ or ``list(m.p.values())``. ``mutable=True``. .. note:: - PyROS also supports uncertain parameters implemented as + PyROS also allows uncertain parameters to be implemented as :class:`~pyomo.core.base.var.Var` objects declared on the deterministic model. This may be convenient for users transitioning to PyROS from - uncertainty quantification workflows. - All :class:`~pyomo.core.base.var.Var` objects representing - uncertain parameters should be fixed. + parameter estimation and/or uncertainty quantification workflows, + in which the uncertain parameters are + often represented by :class:`~pyomo.core.base.var.Var` objects. + Prior to invoking PyROS, + all such :class:`~pyomo.core.base.var.Var` objects should be fixed. PyROS will seek to identify solutions that remain feasible for any From 89ca384ddfa4ee040915e96c0db51c68afd46441 Mon Sep 17 00:00:00 2001 From: jasherma Date: Thu, 30 Jan 2025 13:23:31 -0500 Subject: [PATCH 29/36] Simplify component type check --- pyomo/contrib/pyros/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/pyros/config.py b/pyomo/contrib/pyros/config.py index e3356ebc99f..53053641c36 100644 --- a/pyomo/contrib/pyros/config.py +++ b/pyomo/contrib/pyros/config.py @@ -85,7 +85,7 @@ def uncertain_param_validator(uncertain_obj): "Check that the component has been properly constructed, " "and all entries have been initialized. " ) - if isinstance(uncertain_obj, Param) and not uncertain_obj.mutable: + if uncertain_obj.ctype is Param and not uncertain_obj.mutable: raise ValueError( f"{type(uncertain_obj).__name__} object with name {uncertain_obj.name!r} " "is immutable." From fcfd56ca29bd079c1e540acd46170049df05405f Mon Sep 17 00:00:00 2001 From: jasherma Date: Thu, 30 Jan 2025 13:24:30 -0500 Subject: [PATCH 30/36] Correct argument data type documentation --- pyomo/contrib/pyros/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/pyros/config.py b/pyomo/contrib/pyros/config.py index 53053641c36..ac2b11157e3 100644 --- a/pyomo/contrib/pyros/config.py +++ b/pyomo/contrib/pyros/config.py @@ -99,7 +99,7 @@ def uncertain_param_data_validator(uncertain_obj): Parameters ---------- - uncertain_obj : Param or Var + uncertain_obj : ParamData or VarData Object on which to perform checks. Raises From 8bbb7acd9c61c3db5af8b6089b1fae7ed7119ca4 Mon Sep 17 00:00:00 2001 From: jasherma Date: Thu, 30 Jan 2025 13:26:35 -0500 Subject: [PATCH 31/36] Correct PyROS `ConfigDict` description of argument `uncertain_params` --- pyomo/contrib/pyros/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/pyros/config.py b/pyomo/contrib/pyros/config.py index ac2b11157e3..fb1e2001e8b 100644 --- a/pyomo/contrib/pyros/config.py +++ b/pyomo/contrib/pyros/config.py @@ -548,8 +548,8 @@ def pyros_config(): Uncertain model parameters. Of every constituent `Param` object, the `mutable` attribute must be set to True. - Of every constituent `Var` and `VarData` object, - the domain, declared bounds, and fixing are ignored. + All constituent `Var`/`VarData` objects should be + fixed. """ ), visibility=1, From 9688c3f1a0347899236f063cfcc2ee14ef91069d Mon Sep 17 00:00:00 2001 From: jasherma Date: Thu, 30 Jan 2025 13:30:02 -0500 Subject: [PATCH 32/36] Tweak uncertain parameter substitution logging message --- pyomo/contrib/pyros/util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index d4f2ec3f619..8f471ffbaac 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -1862,8 +1862,6 @@ def setup_working_model(model_data, user_var_partitioning): "Uncertain parameter with name " f"{var.name!r} (relative to the working model clone) " f"is of type {VarData.__name__}. " - f"The user-specified domain, declared bounds, and fixing of " - f"this {VarData.__name__} object will be ignored. " f"A newly declared {ParamData.__name__} object " f"with name {param.name!r} " f"has been substituted for the {VarData.__name__} object " From ad4ceb8c10725d51c2dc9ac3337eaa573eeeb832 Mon Sep 17 00:00:00 2001 From: jasherma Date: Tue, 4 Feb 2025 15:41:19 -0500 Subject: [PATCH 33/36] Add note on assumption for uncertain parameter evaluation --- pyomo/contrib/pyros/util.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 8f471ffbaac..0efacdf3581 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -870,6 +870,10 @@ def _get_uncertain_param_val(var_or_param_data): Get value of VarData/ParamData object that is considered an uncertain parameter. + For any unfixed VarData object, we assume that + the `lower` and `upper` attributes are identical, + so the value of `lower` is returned. + Parameters ---------- var_or_param_data : VarData or ParamData From 148b98afe7d515497bf36873aacfb49aa119a5ee Mon Sep 17 00:00:00 2001 From: jasherma Date: Tue, 4 Feb 2025 15:57:53 -0500 Subject: [PATCH 34/36] Speed up uncertain parameter substitution method --- pyomo/contrib/pyros/util.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 0efacdf3581..21f6ee196d0 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -1767,13 +1767,11 @@ def replace_vars_with_params(block, var_to_param_map): Mapping from VarData objects to be replaced to the ParamData objects to be introduced. """ - # invoke the expression replacement method for each - # individual component type to ensure the substitutions - # are always performed in the named expressions first - for ctype in (Expression, Constraint, Objective): - _replace_vars_in_component_exprs( - block=block, substitution_map=var_to_param_map, ctype=ctype - ) + _replace_vars_in_component_exprs( + block=block, + substitution_map=var_to_param_map, + ctype=(Expression, Constraint, Objective), + ) def setup_working_model(model_data, user_var_partitioning): From daca6d02b16f3898eb1e285e55f28456163eb502 Mon Sep 17 00:00:00 2001 From: jasherma Date: Tue, 4 Feb 2025 15:58:52 -0500 Subject: [PATCH 35/36] Test substituting named expressions in constraints --- pyomo/contrib/pyros/tests/test_preprocessor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyomo/contrib/pyros/tests/test_preprocessor.py b/pyomo/contrib/pyros/tests/test_preprocessor.py index 4a1177a229f..04331dee95c 100644 --- a/pyomo/contrib/pyros/tests/test_preprocessor.py +++ b/pyomo/contrib/pyros/tests/test_preprocessor.py @@ -359,7 +359,7 @@ def build_test_model_data(self): # NAMED EXPRESSIONS: mainly to test # Var -> Param substitution for uncertain params - m.nexpr = Expression(expr=m.q2var + m.x1) + m.nexpr = Expression(expr=log(m.y2) + m.q2var) # EQUALITY CONSTRAINTS m.eq1 = Constraint(expr=m.q * (m.z3 + m.x2) == 0) @@ -371,7 +371,7 @@ def build_test_model_data(self): m.ineq1 = Constraint(expr=(-m.p, m.x1 + m.z1, exp(m.q))) m.ineq2 = Constraint(expr=(0, m.x1 + m.x2, 10)) m.ineq3 = Constraint(expr=(2 * m.q, 2 * (m.z3 + m.y1), 2 * m.q)) - m.ineq4 = Constraint(expr=-m.q <= m.y2**2 + log(m.y2) + m.q2var) + m.ineq4 = Constraint(expr=-m.q <= m.y2**2 + m.nexpr) # out of scope: deactivated m.ineq5 = Constraint(expr=m.y3 <= m.q) @@ -472,13 +472,13 @@ def test_setup_working_model(self): # ensure uncertain Param substitutions carried out properly ublk = model_data.working_model.user_model - self.assertExpressionsEqual(ublk.nexpr.expr, temp_uncertain_param + ublk.x1) self.assertExpressionsEqual( - ublk.inactive_obj.expr, LinearExpression([1, temp_uncertain_param, m.x1]) + ublk.nexpr.expr, log(ublk.y2) + temp_uncertain_param ) self.assertExpressionsEqual( - ublk.ineq4.expr, -ublk.q <= ublk.y2**2 + log(ublk.y2) + temp_uncertain_param + ublk.inactive_obj.expr, LinearExpression([1, temp_uncertain_param, m.x1]) ) + self.assertExpressionsEqual(ublk.ineq4.expr, -ublk.q <= ublk.y2**2 + ublk.nexpr) # other component expressions should remain as declared self.assertExpressionsEqual(ublk.eq1.expr, ublk.q * (ublk.z3 + ublk.x2) == 0) From 7dec651a6a6b11c1e0691be0c3bb1d1af5519107 Mon Sep 17 00:00:00 2001 From: jasherma Date: Tue, 4 Feb 2025 16:08:14 -0500 Subject: [PATCH 36/36] Clarify note on assumption about unfixed Vars --- pyomo/contrib/pyros/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 21f6ee196d0..1f61994bb19 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -872,7 +872,8 @@ def _get_uncertain_param_val(var_or_param_data): For any unfixed VarData object, we assume that the `lower` and `upper` attributes are identical, - so the value of `lower` is returned. + so the value of `lower` is returned in lieu of + the level value. Parameters ----------