From 6d442b24542d524c0ff414a6009408d27deb39df Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 20 Sep 2023 22:38:14 -0400 Subject: [PATCH 001/103] disable ipopt warmstart for feasibility subproblem solver --- pyomo/contrib/mindtpy/algorithm_base_class.py | 2 +- pyomo/contrib/mindtpy/util.py | 21 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index f3877304adb..c254c5d3f3d 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -2574,7 +2574,7 @@ def initialize_subsolvers(self): self.nlp_opt, config.nlp_solver, config ) set_solver_constraint_violation_tolerance( - self.feasibility_nlp_opt, config.nlp_solver, config + self.feasibility_nlp_opt, config.nlp_solver, config, warm_start=False ) self.set_appsi_solver_update_config() diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index e336715cc8f..068cd61aba1 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -566,7 +566,7 @@ def set_solver_mipgap(opt, solver_name, config): opt.options['add_options'].append('option optcr=%s;' % config.mip_solver_mipgap) -def set_solver_constraint_violation_tolerance(opt, solver_name, config): +def set_solver_constraint_violation_tolerance(opt, solver_name, config, warm_start=True): """Set constraint violation tolerance for solvers. Parameters @@ -600,15 +600,16 @@ def set_solver_constraint_violation_tolerance(opt, solver_name, config): opt.options['add_options'].append( 'constr_viol_tol ' + str(config.zero_tolerance) ) - # Ipopt warmstart options - opt.options['add_options'].append( - 'warm_start_init_point yes\n' - 'warm_start_bound_push 1e-9\n' - 'warm_start_bound_frac 1e-9\n' - 'warm_start_slack_bound_frac 1e-9\n' - 'warm_start_slack_bound_push 1e-9\n' - 'warm_start_mult_bound_push 1e-9\n' - ) + if warm_start: + # Ipopt warmstart options + opt.options['add_options'].append( + 'warm_start_init_point yes\n' + 'warm_start_bound_push 1e-9\n' + 'warm_start_bound_frac 1e-9\n' + 'warm_start_slack_bound_frac 1e-9\n' + 'warm_start_slack_bound_push 1e-9\n' + 'warm_start_mult_bound_push 1e-9\n' + ) elif config.nlp_solver_args['solver'] == 'conopt': opt.options['add_options'].append( 'RTNWMA ' + str(config.zero_tolerance) From a6e92c53e63b9febad79e8a19a5230c8c308328a Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 21 Sep 2023 01:44:41 -0400 Subject: [PATCH 002/103] create new copy_var_list_values function --- pyomo/contrib/mindtpy/algorithm_base_class.py | 4 ++- pyomo/contrib/mindtpy/single_tree.py | 3 +- pyomo/contrib/mindtpy/util.py | 29 +++++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index c254c5d3f3d..836df9fff78 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -56,7 +56,6 @@ SuppressInfeasibleWarning, _DoNothing, lower_logger_level_to, - copy_var_list_values, get_main_elapsed_time, time_code, ) @@ -81,6 +80,7 @@ set_solver_mipgap, set_solver_constraint_violation_tolerance, update_solver_timelimit, + copy_var_list_values ) single_tree, single_tree_available = attempt_import('pyomo.contrib.mindtpy.single_tree') @@ -866,12 +866,14 @@ def init_rNLP(self, add_oa_cuts=True): self.rnlp.MindtPy_utils.variable_list, self.mip.MindtPy_utils.variable_list, config, + ignore_integrality=True ) if config.init_strategy == 'FP': copy_var_list_values( self.rnlp.MindtPy_utils.variable_list, self.working_model.MindtPy_utils.variable_list, config, + ignore_integrality=True ) self.add_cuts( dual_values=dual_values, diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 9776920f434..f3be27cbc4c 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -16,9 +16,8 @@ from pyomo.repn import generate_standard_repn import pyomo.core.expr as EXPR from math import copysign -from pyomo.contrib.mindtpy.util import get_integer_solution +from pyomo.contrib.mindtpy.util import get_integer_solution, copy_var_list_values from pyomo.contrib.gdpopt.util import ( - copy_var_list_values, get_main_elapsed_time, time_code, ) diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 068cd61aba1..59490248e49 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -23,6 +23,7 @@ RangeSet, ConstraintList, TransformationFactory, + value ) from pyomo.repn import generate_standard_repn from pyomo.contrib.mcpp.pyomo_mcpp import mcpp_available, McCormick @@ -964,3 +965,31 @@ def generate_norm_constraint(fp_nlp_model, mip_model, config): mip_model.MindtPy_utils.discrete_variable_list, ): fp_nlp_model.norm_constraint.add(nlp_var - mip_var.value <= rhs) + +def copy_var_list_values(from_list, to_list, config, + skip_stale=False, skip_fixed=True, + ignore_integrality=False): + """Copy variable values from one list to another. + Rounds to Binary/Integer if necessary + Sets to zero for NonNegativeReals if necessary + """ + for v_from, v_to in zip(from_list, to_list): + if skip_stale and v_from.stale: + continue # Skip stale variable values. + if skip_fixed and v_to.is_fixed(): + continue # Skip fixed variables. + var_val = value(v_from, exception=False) + rounded_val = int(round(var_val)) + if var_val in v_to.domain: + v_to.set_value(value(v_from, exception=False)) + elif ignore_integrality and v_to.is_integer(): + v_to.set_value(value(v_from, exception=False)) + elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: + v_to.set_value(0) + elif v_to.is_integer() and (math.fabs(var_val - rounded_val) <= + config.integer_tolerance): + print('var_val', var_val) + v_to.pprint() + v_to.set_value(rounded_val) + else: + raise From f766c0a9d6f66da8fea8b11f277bb719ef94f48f Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 21 Sep 2023 19:23:03 -0400 Subject: [PATCH 003/103] update log format --- pyomo/contrib/mindtpy/algorithm_base_class.py | 30 +++++++++++++++++-- pyomo/contrib/mindtpy/util.py | 2 -- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index 836df9fff78..514c2edaedb 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -124,6 +124,9 @@ def __init__(self, **kwds): self.fixed_nlp_log_formatter = ( '{:1}{:>9} {:>15} {:>15g} {:>12g} {:>12g} {:>7.2%} {:>7.2f}' ) + self.infeasible_fixed_nlp_log_formatter = ( + '{:1}{:>9} {:>15} {:>15} {:>12g} {:>12g} {:>7.2%} {:>7.2f}' + ) self.log_note_formatter = ' {:>9} {:>15} {:>15}' # Flag indicating whether the solution improved in the past @@ -1210,7 +1213,18 @@ def handle_subproblem_infeasible(self, fixed_nlp, cb_opt=None): # TODO try something else? Reinitialize with different initial # value? config = self.config - config.logger.info('NLP subproblem was locally infeasible.') + config.logger.info( + self.infeasible_fixed_nlp_log_formatter.format( + ' ', + self.nlp_iter, + 'Fixed NLP', + 'Infeasible', + self.primal_bound, + self.dual_bound, + self.rel_gap, + get_main_elapsed_time(self.timing), + ) + ) self.nlp_infeasible_counter += 1 if config.calculate_dual_at_solution: for c in fixed_nlp.MindtPy_utils.constraint_list: @@ -1232,7 +1246,7 @@ def handle_subproblem_infeasible(self, fixed_nlp, cb_opt=None): # elif var.has_lb() and abs(value(var) - var.lb) < config.absolute_bound_tolerance: # fixed_nlp.ipopt_zU_out[var] = -1 - config.logger.info('Solving feasibility problem') + # config.logger.info('Solving feasibility problem') feas_subproblem, feas_subproblem_results = self.solve_feasibility_subproblem() # TODO: do we really need this? if self.should_terminate: @@ -1366,6 +1380,18 @@ def solve_feasibility_subproblem(self): self.handle_feasibility_subproblem_tc( feas_soln.solver.termination_condition, MindtPy ) + config.logger.info( + self.fixed_nlp_log_formatter.format( + ' ', + self.nlp_iter, + 'Feasibility NLP', + value(feas_subproblem.MindtPy_utils.feas_obj), + self.primal_bound, + self.dual_bound, + self.rel_gap, + get_main_elapsed_time(self.timing), + ) + ) MindtPy.feas_opt.deactivate() for constr in MindtPy.nonlinear_constraint_list: constr.activate() diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 59490248e49..4a4b77767a9 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -988,8 +988,6 @@ def copy_var_list_values(from_list, to_list, config, v_to.set_value(0) elif v_to.is_integer() and (math.fabs(var_val - rounded_val) <= config.integer_tolerance): - print('var_val', var_val) - v_to.pprint() v_to.set_value(rounded_val) else: raise From 83069253086f5199c35ee22aadf1b1ea85fdf891 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Sat, 23 Sep 2023 15:08:00 -0400 Subject: [PATCH 004/103] add update_solver_timelimit --- pyomo/contrib/mindtpy/algorithm_base_class.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index 514c2edaedb..0eb602bdf7e 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -1618,6 +1618,7 @@ def solve_main(self): # setup main problem self.setup_main() mip_args = self.set_up_mip_solver() + update_solver_timelimit(self.mip_opt, config.mip_solver, self.timing, config) try: main_mip_results = self.mip_opt.solve( @@ -1675,6 +1676,9 @@ def solve_fp_main(self): config = self.config self.setup_fp_main() mip_args = self.set_up_mip_solver() + update_solver_timelimit( + self.mip_opt, config.mip_solver, self.timing, config + ) main_mip_results = self.mip_opt.solve( self.mip, From 816e3d66492da945e01bad7a906810b400e90f71 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Mon, 9 Oct 2023 18:16:09 -0400 Subject: [PATCH 005/103] handle appsi solver unbounded situation --- pyomo/contrib/mindtpy/algorithm_base_class.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index 0eb602bdf7e..89575f5c4f2 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -1644,7 +1644,8 @@ def solve_main(self): "No-good cuts are added and GOA algorithm doesn't converge within the time limit. " 'No integer solution is found, so the CPLEX solver will report an error status. ' ) - return None, None + # Value error will be raised if the MIP problem is unbounded and appsi solver is used when loading solutions. Although the problem is unbounded, a valid result is provided and we do not return None to let the algorithm continue. + return self.mip, main_mip_results if config.solution_pool: main_mip_results._solver_model = self.mip_opt._solver_model main_mip_results._pyomo_var_to_solver_var_map = ( From 5c47040d5b0fee4bb54656e49dff54f670ec6680 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Tue, 10 Oct 2023 15:57:39 -0400 Subject: [PATCH 006/103] add skip_validation when ignore integrality --- pyomo/contrib/mindtpy/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 4a4b77767a9..4dfb912e611 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -983,7 +983,7 @@ def copy_var_list_values(from_list, to_list, config, if var_val in v_to.domain: v_to.set_value(value(v_from, exception=False)) elif ignore_integrality and v_to.is_integer(): - v_to.set_value(value(v_from, exception=False)) + v_to.set_value(value(v_from, exception=False), skip_validation=True) elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: v_to.set_value(0) elif v_to.is_integer() and (math.fabs(var_val - rounded_val) <= From a767f281acd08629aea99f6c616e59b11372d2f5 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Tue, 10 Oct 2023 16:28:58 -0400 Subject: [PATCH 007/103] add special handle for rnlp infeasible --- pyomo/contrib/mindtpy/algorithm_base_class.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index 89575f5c4f2..f6192ad4687 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -829,6 +829,23 @@ def init_rNLP(self, add_oa_cuts=True): if len(results.solution) > 0: self.rnlp.solutions.load_from(results) subprob_terminate_cond = results.solver.termination_condition + + # Sometimes, the NLP solver might be trapped in a infeasible solution if the objective function is nonlinear and partition_obj_nonlinear_terms is True. If this happens, we will use the original objective function instead. + if subprob_terminate_cond == tc.infeasible and config.partition_obj_nonlinear_terms: + config.logger.info( + 'Initial relaxed NLP problem is infeasible. This might be related to partition_obj_nonlinear_terms. Try to solve it again without partitioning nonlinear objective function.') + self.rnlp.MindtPy_utils.objective.deactivate() + self.rnlp.MindtPy_utils.objective_list[0].activate() + results = self.nlp_opt.solve( + self.rnlp, + tee=config.nlp_solver_tee, + load_solutions=config.load_solutions, + **nlp_args, + ) + if len(results.solution) > 0: + self.rnlp.solutions.load_from(results) + subprob_terminate_cond = results.solver.termination_condition + if subprob_terminate_cond in {tc.optimal, tc.feasible, tc.locallyOptimal}: main_objective = MindtPy.objective_list[-1] if subprob_terminate_cond == tc.optimal: From a30cf823fbe9e370f0feb2c7ce5ed659d5db112e Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Tue, 10 Oct 2023 16:30:08 -0400 Subject: [PATCH 008/103] fix bug --- pyomo/contrib/mindtpy/algorithm_base_class.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index f6192ad4687..c254a8db72b 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -1662,7 +1662,10 @@ def solve_main(self): 'No integer solution is found, so the CPLEX solver will report an error status. ' ) # Value error will be raised if the MIP problem is unbounded and appsi solver is used when loading solutions. Although the problem is unbounded, a valid result is provided and we do not return None to let the algorithm continue. - return self.mip, main_mip_results + if 'main_mip_results' in dir(): + return self.mip, main_mip_results + else: + return None, None if config.solution_pool: main_mip_results._solver_model = self.mip_opt._solver_model main_mip_results._pyomo_var_to_solver_var_map = ( From 2a6f1d7c9b41f3d018a6095c86d4d5f54dd2bbdf Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 11 Oct 2023 23:12:28 -0400 Subject: [PATCH 009/103] add comments --- pyomo/contrib/mindtpy/single_tree.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index f3be27cbc4c..8b8e171c577 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -707,8 +707,8 @@ def __call__(self): # Reference: https://www.ibm.com/docs/en/icos/22.1.1?topic=SSSA5P_22.1.1/ilog.odms.cplex.help/refpythoncplex/html/cplex.callbacks.SolutionSource-class.htm # Another solution source is user_solution = 118, but it will not be encountered in LazyConstraintCallback. - config.logger.debug( - "Solution source: %s (111 node_solution, 117 heuristic_solution, 119 mipstart_solution)".format( + config.logger.info( + "Solution source: {} (111 node_solution, 117 heuristic_solution, 119 mipstart_solution)".format( self.get_solution_source() ) ) @@ -717,6 +717,7 @@ def __call__(self): # Lazy constraints separated when processing a MIP start will be discarded after that MIP start has been processed. # This means that the callback may have to separate the same constraint again for the next MIP start or for a solution that is found later in the solution process. # https://www.ibm.com/docs/en/icos/22.1.1?topic=SSSA5P_22.1.1/ilog.odms.cplex.help/refpythoncplex/html/cplex.callbacks.LazyConstraintCallback-class.htm + # For the MINLP3_simple example, all the solutions are obtained from mip_start (solution source). Therefore, it will not go to a branch and bound process.Cause an error output. if ( self.get_solution_source() != cplex.callbacks.SolutionSource.mipstart_solution From 6bcdadbfcdb9b81abe987aed665c5137c45b7c07 Mon Sep 17 00:00:00 2001 From: Zedong Peng Date: Thu, 26 Oct 2023 21:12:46 -0400 Subject: [PATCH 010/103] fix bug --- pyomo/contrib/mindtpy/algorithm_base_class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index c254a8db72b..c0afea58a99 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -831,7 +831,7 @@ def init_rNLP(self, add_oa_cuts=True): subprob_terminate_cond = results.solver.termination_condition # Sometimes, the NLP solver might be trapped in a infeasible solution if the objective function is nonlinear and partition_obj_nonlinear_terms is True. If this happens, we will use the original objective function instead. - if subprob_terminate_cond == tc.infeasible and config.partition_obj_nonlinear_terms: + if subprob_terminate_cond == tc.infeasible and config.partition_obj_nonlinear_terms and self.rnlp.MindtPy_utils.objective_list[0].expr.polynomial_degree() not in self.mip_objective_polynomial_degree: config.logger.info( 'Initial relaxed NLP problem is infeasible. This might be related to partition_obj_nonlinear_terms. Try to solve it again without partitioning nonlinear objective function.') self.rnlp.MindtPy_utils.objective.deactivate() From a66f955500686fef09d8605571ef6f5c238101f8 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Sun, 29 Oct 2023 15:04:27 -0400 Subject: [PATCH 011/103] improve copy_var_list_values function --- pyomo/contrib/mindtpy/single_tree.py | 76 ++++++++++++------------- pyomo/contrib/mindtpy/util.py | 83 +++++++++++++++------------- 2 files changed, 83 insertions(+), 76 deletions(-) diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 8b8e171c577..2174d093009 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -24,6 +24,7 @@ from pyomo.opt import TerminationCondition as tc from pyomo.core import minimize, value from pyomo.core.expr import identify_variables +import math cplex, cplex_available = attempt_import('cplex') @@ -41,7 +42,6 @@ def copy_lazy_var_list_values( config, skip_stale=False, skip_fixed=True, - ignore_integrality=False, ): """This function copies variable values from one list to another. @@ -71,43 +71,43 @@ def copy_lazy_var_list_values( if skip_fixed and v_to.is_fixed(): continue # Skip fixed variables. v_val = self.get_values(opt._pyomo_var_to_solver_var_map[v_from]) - try: - # We don't want to trigger the reset of the global stale - # indicator, so we will set this variable to be "stale", - # knowing that set_value will switch it back to "not - # stale" - v_to.stale = True - # NOTE: PEP 2180 changes the var behavior so that domain - # / bounds violations no longer generate exceptions (and - # instead log warnings). This means that the following - # will always succeed and the ValueError should never be - # raised. - v_to.set_value(v_val, skip_validation=True) - except ValueError as e: - # Snap the value to the bounds - config.logger.error(e) - if ( - v_to.has_lb() - and v_val < v_to.lb - and v_to.lb - v_val <= config.variable_tolerance - ): - v_to.set_value(v_to.lb, skip_validation=True) - elif ( - v_to.has_ub() - and v_val > v_to.ub - and v_val - v_to.ub <= config.variable_tolerance - ): - v_to.set_value(v_to.ub, skip_validation=True) - # ... or the nearest integer - elif v_to.is_integer(): - rounded_val = int(round(v_val)) - if ( - ignore_integrality - or abs(v_val - rounded_val) <= config.integer_tolerance - ) and rounded_val in v_to.domain: - v_to.set_value(rounded_val, skip_validation=True) - else: - raise + rounded_val = int(round(v_val)) + # We don't want to trigger the reset of the global stale + # indicator, so we will set this variable to be "stale", + # knowing that set_value will switch it back to "not + # stale" + v_to.stale = True + # NOTE: PEP 2180 changes the var behavior so that domain + # / bounds violations no longer generate exceptions (and + # instead log warnings). This means that the following + # will always succeed and the ValueError should never be + # raised. + if v_val in v_to.domain \ + and not ((v_to.has_lb() and v_val < v_to.lb)) \ + and not ((v_to.has_ub() and v_val > v_to.ub)): + v_to.set_value(v_val) + # Snap the value to the bounds + # TODO: check the performance of + # v_to.lb - v_val <= config.variable_tolerance + elif ( + v_to.has_lb() + and v_val < v_to.lb + # and v_to.lb - v_val <= config.variable_tolerance + ): + v_to.set_value(v_to.lb) + elif ( + v_to.has_ub() + and v_val > v_to.ub + # and v_val - v_to.ub <= config.variable_tolerance + ): + v_to.set_value(v_to.ub) + # ... or the nearest integer + elif v_to.is_integer() and math.fabs(v_val - rounded_val) <= config.integer_tolerance: # and rounded_val in v_to.domain: + v_to.set_value(rounded_val) + elif abs(v_val) <= config.zero_tolerance and 0 in v_to.domain: + v_to.set_value(0) + else: + raise ValueError('copy_lazy_var_list_values failed.') def add_lazy_oa_cuts( self, diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 4dfb912e611..da1534b49ac 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -684,41 +684,42 @@ def copy_var_list_values_from_solution_pool( Whether to ignore the integrality of integer variables, by default False. """ for v_from, v_to in zip(from_list, to_list): - try: - if config.mip_solver == 'cplex_persistent': - var_val = solver_model.solution.pool.get_values( - solution_name, var_map[v_from] - ) - elif config.mip_solver == 'gurobi_persistent': - solver_model.setParam(gurobipy.GRB.Param.SolutionNumber, solution_name) - var_val = var_map[v_from].Xn - # We don't want to trigger the reset of the global stale - # indicator, so we will set this variable to be "stale", - # knowing that set_value will switch it back to "not - # stale" - v_to.stale = True - # NOTE: PEP 2180 changes the var behavior so that domain / - # bounds violations no longer generate exceptions (and - # instead log warnings). This means that the following will - # always succeed and the ValueError should never be raised. + if config.mip_solver == 'cplex_persistent': + var_val = solver_model.solution.pool.get_values( + solution_name, var_map[v_from] + ) + elif config.mip_solver == 'gurobi_persistent': + solver_model.setParam(gurobipy.GRB.Param.SolutionNumber, solution_name) + var_val = var_map[v_from].Xn + # We don't want to trigger the reset of the global stale + # indicator, so we will set this variable to be "stale", + # knowing that set_value will switch it back to "not + # stale" + v_to.stale = True + rounded_val = int(round(var_val)) + # NOTE: PEP 2180 changes the var behavior so that domain / + # bounds violations no longer generate exceptions (and + # instead log warnings). This means that the following will + # always succeed and the ValueError should never be raised. + if var_val in v_to.domain \ + and not ((v_to.has_lb() and var_val < v_to.lb)) \ + and not ((v_to.has_ub() and var_val > v_to.ub)): v_to.set_value(var_val, skip_validation=True) - except ValueError as e: - config.logger.error(e) - rounded_val = int(round(var_val)) - # Check to see if this is just a tolerance issue - if ignore_integrality and v_to.is_integer(): - v_to.set_value(var_val, skip_validation=True) - elif v_to.is_integer() and ( - abs(var_val - rounded_val) <= config.integer_tolerance - ): - v_to.set_value(rounded_val, skip_validation=True) - elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: - v_to.set_value(0, skip_validation=True) - else: - config.logger.error( - 'Unknown validation domain error setting variable %s' % (v_to.name,) - ) - raise + elif v_to.has_lb() and var_val < v_to.lb: + v_to.set_value(v_to.lb) + elif v_to.has_ub() and var_val > v_to.ub: + v_to.set_value(v_to.ub) + # Check to see if this is just a tolerance issue + elif ignore_integrality and v_to.is_integer(): + v_to.set_value(var_val, skip_validation=True) + elif v_to.is_integer() and ( + abs(var_val - rounded_val) <= config.integer_tolerance + ): + v_to.set_value(rounded_val, skip_validation=True) + elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: + v_to.set_value(0, skip_validation=True) + else: + raise ValueError("copy_var_list_values_from_solution_pool failed.") class GurobiPersistent4MindtPy(GurobiPersistent): @@ -980,14 +981,20 @@ def copy_var_list_values(from_list, to_list, config, continue # Skip fixed variables. var_val = value(v_from, exception=False) rounded_val = int(round(var_val)) - if var_val in v_to.domain: + if var_val in v_to.domain \ + and not ((v_to.has_lb() and var_val < v_to.lb)) \ + and not ((v_to.has_ub() and var_val > v_to.ub)): v_to.set_value(value(v_from, exception=False)) + elif v_to.has_lb() and var_val < v_to.lb: + v_to.set_value(v_to.lb) + elif v_to.has_ub() and var_val > v_to.ub: + v_to.set_value(v_to.ub) elif ignore_integrality and v_to.is_integer(): v_to.set_value(value(v_from, exception=False), skip_validation=True) - elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: - v_to.set_value(0) elif v_to.is_integer() and (math.fabs(var_val - rounded_val) <= config.integer_tolerance): v_to.set_value(rounded_val) + elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: + v_to.set_value(0) else: - raise + raise ValueError("copy_var_list_values failed.") From cefd4a66a06711e90005d20cd470efca762754a6 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Sun, 29 Oct 2023 15:12:02 -0400 Subject: [PATCH 012/103] fix FP bug --- pyomo/contrib/mindtpy/algorithm_base_class.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index c254a8db72b..3e53559b3df 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -2384,6 +2384,7 @@ def handle_fp_subproblem_optimal(self, fp_nlp): fp_nlp.MindtPy_utils.variable_list, self.working_model.MindtPy_utils.variable_list, self.config, + ignore_integrality=True ) add_orthogonality_cuts(self.working_model, self.mip, self.config) From 905503b13907a610a1e7f9d8620ee88127551ec5 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 1 Nov 2023 00:42:53 -0400 Subject: [PATCH 013/103] fix gurobi single tree termination check bug --- pyomo/contrib/mindtpy/single_tree.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 2174d093009..9595a9fc9be 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -910,19 +910,7 @@ def LazyOACallback_gurobi(cb_m, cb_opt, cb_where, mindtpy_solver, config): if mindtpy_solver.dual_bound != mindtpy_solver.dual_bound_progress[0]: mindtpy_solver.add_regularization() - if ( - abs(mindtpy_solver.primal_bound - mindtpy_solver.dual_bound) - <= config.absolute_bound_tolerance - ): - config.logger.info( - 'MindtPy exiting on bound convergence. ' - '|Primal Bound: {} - Dual Bound: {}| <= (absolute tolerance {}) \n'.format( - mindtpy_solver.primal_bound, - mindtpy_solver.dual_bound, - config.absolute_bound_tolerance, - ) - ) - mindtpy_solver.results.solver.termination_condition = tc.optimal + if mindtpy_solver.bounds_converged() or mindtpy_solver.reached_time_limit(): cb_opt._solver_model.terminate() return From 3b01104f586f9a33d1386ff89c5c3e57b3ae7fc7 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 1 Nov 2023 21:24:39 -0400 Subject: [PATCH 014/103] fix Gurobi single tree cycle handling --- pyomo/contrib/mindtpy/algorithm_base_class.py | 3 +++ pyomo/contrib/mindtpy/single_tree.py | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index a0216b5d054..33b2f2c1d04 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -106,6 +106,8 @@ def __init__(self, **kwds): self.curr_int_sol = [] self.should_terminate = False self.integer_list = [] + # dictionary {integer solution (list): cuts index (list)} + self.int_sol_2_cuts_ind = dict() # Set up iteration counters self.nlp_iter = 0 @@ -794,6 +796,7 @@ def MindtPy_initialization(self): self.integer_list.append(self.curr_int_sol) fixed_nlp, fixed_nlp_result = self.solve_subproblem() self.handle_nlp_subproblem_tc(fixed_nlp, fixed_nlp_result) + self.int_sol_2_cuts_ind[self.curr_int_sol] = list(range(1, len(self.mip.MindtPy_utils.cuts.oa_cuts) + 1)) elif config.init_strategy == 'FP': self.init_rNLP() self.fp_loop() diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 9595a9fc9be..dacde73a79e 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -941,15 +941,26 @@ def LazyOACallback_gurobi(cb_m, cb_opt, cb_where, mindtpy_solver, config): ) return elif config.strategy == 'OA': + # Refer to the official document of GUROBI. + # Your callback should be prepared to cut off solutions that violate any of your lazy constraints, including those that have already been added. Node solutions will usually respect previously added lazy constraints, but not always. + # https://www.gurobi.com/documentation/current/refman/cs_cb_addlazy.html + # If this happens, MindtPy will look for the index of corresponding cuts, instead of solving the fixed-NLP again. + for ind in mindtpy_solver.int_sol_2_cuts_ind[mindtpy_solver.curr_int_sol]: + cb_opt.cbLazy(mindtpy_solver.mip.MindtPy_utils.cuts.oa_cuts[ind]) return else: mindtpy_solver.integer_list.append(mindtpy_solver.curr_int_sol) + if config.strategy == 'OA': + cut_ind = len(mindtpy_solver.mip.MindtPy_utils.cuts.oa_cuts) # solve subproblem # The constraint linearization happens in the handlers fixed_nlp, fixed_nlp_result = mindtpy_solver.solve_subproblem() mindtpy_solver.handle_nlp_subproblem_tc(fixed_nlp, fixed_nlp_result, cb_opt) + if config.strategy == 'OA': + # store the cut index corresponding to current integer solution. + mindtpy_solver.int_sol_2_cuts_ind[mindtpy_solver.curr_int_sol] = list(range(cut_ind + 1, len(mindtpy_solver.mip.MindtPy_utils.cuts.oa_cuts) + 1)) def handle_lazy_main_feasible_solution_gurobi(cb_m, cb_opt, mindtpy_solver, config): From 906fff7d18e9cc98c6a7d7e0a1b9d4657aaa9be9 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Mon, 13 Nov 2023 10:59:38 -0500 Subject: [PATCH 015/103] black format --- pyomo/contrib/mindtpy/algorithm_base_class.py | 26 ++++++++----- pyomo/contrib/mindtpy/single_tree.py | 38 +++++++++--------- pyomo/contrib/mindtpy/util.py | 39 ++++++++++++------- 3 files changed, 62 insertions(+), 41 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index 33b2f2c1d04..9771c04fc62 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -80,7 +80,7 @@ set_solver_mipgap, set_solver_constraint_violation_tolerance, update_solver_timelimit, - copy_var_list_values + copy_var_list_values, ) single_tree, single_tree_available = attempt_import('pyomo.contrib.mindtpy.single_tree') @@ -796,7 +796,9 @@ def MindtPy_initialization(self): self.integer_list.append(self.curr_int_sol) fixed_nlp, fixed_nlp_result = self.solve_subproblem() self.handle_nlp_subproblem_tc(fixed_nlp, fixed_nlp_result) - self.int_sol_2_cuts_ind[self.curr_int_sol] = list(range(1, len(self.mip.MindtPy_utils.cuts.oa_cuts) + 1)) + self.int_sol_2_cuts_ind[self.curr_int_sol] = list( + range(1, len(self.mip.MindtPy_utils.cuts.oa_cuts) + 1) + ) elif config.init_strategy == 'FP': self.init_rNLP() self.fp_loop() @@ -834,9 +836,15 @@ def init_rNLP(self, add_oa_cuts=True): subprob_terminate_cond = results.solver.termination_condition # Sometimes, the NLP solver might be trapped in a infeasible solution if the objective function is nonlinear and partition_obj_nonlinear_terms is True. If this happens, we will use the original objective function instead. - if subprob_terminate_cond == tc.infeasible and config.partition_obj_nonlinear_terms and self.rnlp.MindtPy_utils.objective_list[0].expr.polynomial_degree() not in self.mip_objective_polynomial_degree: + if ( + subprob_terminate_cond == tc.infeasible + and config.partition_obj_nonlinear_terms + and self.rnlp.MindtPy_utils.objective_list[0].expr.polynomial_degree() + not in self.mip_objective_polynomial_degree + ): config.logger.info( - 'Initial relaxed NLP problem is infeasible. This might be related to partition_obj_nonlinear_terms. Try to solve it again without partitioning nonlinear objective function.') + 'Initial relaxed NLP problem is infeasible. This might be related to partition_obj_nonlinear_terms. Try to solve it again without partitioning nonlinear objective function.' + ) self.rnlp.MindtPy_utils.objective.deactivate() self.rnlp.MindtPy_utils.objective_list[0].activate() results = self.nlp_opt.solve( @@ -889,14 +897,14 @@ def init_rNLP(self, add_oa_cuts=True): self.rnlp.MindtPy_utils.variable_list, self.mip.MindtPy_utils.variable_list, config, - ignore_integrality=True + ignore_integrality=True, ) if config.init_strategy == 'FP': copy_var_list_values( self.rnlp.MindtPy_utils.variable_list, self.working_model.MindtPy_utils.variable_list, config, - ignore_integrality=True + ignore_integrality=True, ) self.add_cuts( dual_values=dual_values, @@ -1700,9 +1708,7 @@ def solve_fp_main(self): config = self.config self.setup_fp_main() mip_args = self.set_up_mip_solver() - update_solver_timelimit( - self.mip_opt, config.mip_solver, self.timing, config - ) + update_solver_timelimit(self.mip_opt, config.mip_solver, self.timing, config) main_mip_results = self.mip_opt.solve( self.mip, @@ -2387,7 +2393,7 @@ def handle_fp_subproblem_optimal(self, fp_nlp): fp_nlp.MindtPy_utils.variable_list, self.working_model.MindtPy_utils.variable_list, self.config, - ignore_integrality=True + ignore_integrality=True, ) add_orthogonality_cuts(self.working_model, self.mip, self.config) diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index dacde73a79e..66435c2587f 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -17,10 +17,7 @@ import pyomo.core.expr as EXPR from math import copysign from pyomo.contrib.mindtpy.util import get_integer_solution, copy_var_list_values -from pyomo.contrib.gdpopt.util import ( - get_main_elapsed_time, - time_code, -) +from pyomo.contrib.gdpopt.util import get_main_elapsed_time, time_code from pyomo.opt import TerminationCondition as tc from pyomo.core import minimize, value from pyomo.core.expr import identify_variables @@ -35,13 +32,7 @@ class LazyOACallback_cplex( """Inherent class in CPLEX to call Lazy callback.""" def copy_lazy_var_list_values( - self, - opt, - from_list, - to_list, - config, - skip_stale=False, - skip_fixed=True, + self, opt, from_list, to_list, config, skip_stale=False, skip_fixed=True ): """This function copies variable values from one list to another. @@ -82,12 +73,14 @@ def copy_lazy_var_list_values( # instead log warnings). This means that the following # will always succeed and the ValueError should never be # raised. - if v_val in v_to.domain \ - and not ((v_to.has_lb() and v_val < v_to.lb)) \ - and not ((v_to.has_ub() and v_val > v_to.ub)): + if ( + v_val in v_to.domain + and not ((v_to.has_lb() and v_val < v_to.lb)) + and not ((v_to.has_ub() and v_val > v_to.ub)) + ): v_to.set_value(v_val) # Snap the value to the bounds - # TODO: check the performance of + # TODO: check the performance of # v_to.lb - v_val <= config.variable_tolerance elif ( v_to.has_lb() @@ -102,7 +95,10 @@ def copy_lazy_var_list_values( ): v_to.set_value(v_to.ub) # ... or the nearest integer - elif v_to.is_integer() and math.fabs(v_val - rounded_val) <= config.integer_tolerance: # and rounded_val in v_to.domain: + elif ( + v_to.is_integer() + and math.fabs(v_val - rounded_val) <= config.integer_tolerance + ): # and rounded_val in v_to.domain: v_to.set_value(rounded_val) elif abs(v_val) <= config.zero_tolerance and 0 in v_to.domain: v_to.set_value(0) @@ -945,7 +941,9 @@ def LazyOACallback_gurobi(cb_m, cb_opt, cb_where, mindtpy_solver, config): # Your callback should be prepared to cut off solutions that violate any of your lazy constraints, including those that have already been added. Node solutions will usually respect previously added lazy constraints, but not always. # https://www.gurobi.com/documentation/current/refman/cs_cb_addlazy.html # If this happens, MindtPy will look for the index of corresponding cuts, instead of solving the fixed-NLP again. - for ind in mindtpy_solver.int_sol_2_cuts_ind[mindtpy_solver.curr_int_sol]: + for ind in mindtpy_solver.int_sol_2_cuts_ind[ + mindtpy_solver.curr_int_sol + ]: cb_opt.cbLazy(mindtpy_solver.mip.MindtPy_utils.cuts.oa_cuts[ind]) return else: @@ -960,7 +958,11 @@ def LazyOACallback_gurobi(cb_m, cb_opt, cb_where, mindtpy_solver, config): mindtpy_solver.handle_nlp_subproblem_tc(fixed_nlp, fixed_nlp_result, cb_opt) if config.strategy == 'OA': # store the cut index corresponding to current integer solution. - mindtpy_solver.int_sol_2_cuts_ind[mindtpy_solver.curr_int_sol] = list(range(cut_ind + 1, len(mindtpy_solver.mip.MindtPy_utils.cuts.oa_cuts) + 1)) + mindtpy_solver.int_sol_2_cuts_ind[mindtpy_solver.curr_int_sol] = list( + range( + cut_ind + 1, len(mindtpy_solver.mip.MindtPy_utils.cuts.oa_cuts) + 1 + ) + ) def handle_lazy_main_feasible_solution_gurobi(cb_m, cb_opt, mindtpy_solver, config): diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index da1534b49ac..48c8aab31c4 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -23,7 +23,7 @@ RangeSet, ConstraintList, TransformationFactory, - value + value, ) from pyomo.repn import generate_standard_repn from pyomo.contrib.mcpp.pyomo_mcpp import mcpp_available, McCormick @@ -567,7 +567,9 @@ def set_solver_mipgap(opt, solver_name, config): opt.options['add_options'].append('option optcr=%s;' % config.mip_solver_mipgap) -def set_solver_constraint_violation_tolerance(opt, solver_name, config, warm_start=True): +def set_solver_constraint_violation_tolerance( + opt, solver_name, config, warm_start=True +): """Set constraint violation tolerance for solvers. Parameters @@ -701,9 +703,11 @@ def copy_var_list_values_from_solution_pool( # bounds violations no longer generate exceptions (and # instead log warnings). This means that the following will # always succeed and the ValueError should never be raised. - if var_val in v_to.domain \ - and not ((v_to.has_lb() and var_val < v_to.lb)) \ - and not ((v_to.has_ub() and var_val > v_to.ub)): + if ( + var_val in v_to.domain + and not ((v_to.has_lb() and var_val < v_to.lb)) + and not ((v_to.has_ub() and var_val > v_to.ub)) + ): v_to.set_value(var_val, skip_validation=True) elif v_to.has_lb() and var_val < v_to.lb: v_to.set_value(v_to.lb) @@ -967,9 +971,15 @@ def generate_norm_constraint(fp_nlp_model, mip_model, config): ): fp_nlp_model.norm_constraint.add(nlp_var - mip_var.value <= rhs) -def copy_var_list_values(from_list, to_list, config, - skip_stale=False, skip_fixed=True, - ignore_integrality=False): + +def copy_var_list_values( + from_list, + to_list, + config, + skip_stale=False, + skip_fixed=True, + ignore_integrality=False, +): """Copy variable values from one list to another. Rounds to Binary/Integer if necessary Sets to zero for NonNegativeReals if necessary @@ -981,9 +991,11 @@ def copy_var_list_values(from_list, to_list, config, continue # Skip fixed variables. var_val = value(v_from, exception=False) rounded_val = int(round(var_val)) - if var_val in v_to.domain \ - and not ((v_to.has_lb() and var_val < v_to.lb)) \ - and not ((v_to.has_ub() and var_val > v_to.ub)): + if ( + var_val in v_to.domain + and not ((v_to.has_lb() and var_val < v_to.lb)) + and not ((v_to.has_ub() and var_val > v_to.ub)) + ): v_to.set_value(value(v_from, exception=False)) elif v_to.has_lb() and var_val < v_to.lb: v_to.set_value(v_to.lb) @@ -991,8 +1003,9 @@ def copy_var_list_values(from_list, to_list, config, v_to.set_value(v_to.ub) elif ignore_integrality and v_to.is_integer(): v_to.set_value(value(v_from, exception=False), skip_validation=True) - elif v_to.is_integer() and (math.fabs(var_val - rounded_val) <= - config.integer_tolerance): + elif v_to.is_integer() and ( + math.fabs(var_val - rounded_val) <= config.integer_tolerance + ): v_to.set_value(rounded_val) elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: v_to.set_value(0) From 0e06806cd7b9515e3b641feae79a681dbf7bd1d4 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 20 Nov 2023 16:41:04 -0700 Subject: [PATCH 016/103] Adding all_different and count_if expression nodes to the logical expression system --- pyomo/core/__init__.py | 2 + pyomo/core/expr/__init__.py | 2 + pyomo/core/expr/logical_expr.py | 73 ++++++++++++++++++- .../tests/unit/test_logical_expr_expanded.py | 46 +++++++++++- pyomo/environ/__init__.py | 2 + 5 files changed, 121 insertions(+), 4 deletions(-) diff --git a/pyomo/core/__init__.py b/pyomo/core/__init__.py index 5cbebcee9ec..b119c6357d0 100644 --- a/pyomo/core/__init__.py +++ b/pyomo/core/__init__.py @@ -33,6 +33,8 @@ exactly, atleast, atmost, + all_different, + count_if, implies, lnot, xor, diff --git a/pyomo/core/expr/__init__.py b/pyomo/core/expr/__init__.py index 5e30fceeeaa..de2228189f9 100644 --- a/pyomo/core/expr/__init__.py +++ b/pyomo/core/expr/__init__.py @@ -79,6 +79,8 @@ exactly, atleast, atmost, + all_different, + count_if, implies, ) from .numeric_expr import ( diff --git a/pyomo/core/expr/logical_expr.py b/pyomo/core/expr/logical_expr.py index e5a2f411a6e..2b261278ee9 100644 --- a/pyomo/core/expr/logical_expr.py +++ b/pyomo/core/expr/logical_expr.py @@ -10,10 +10,8 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -from __future__ import division - import types -from itertools import islice +from itertools import combinations, islice import logging import traceback @@ -37,6 +35,7 @@ from .base import ExpressionBase from .boolean_value import BooleanValue, BooleanConstant from .expr_common import _and, _or, _equiv, _inv, _xor, _impl, ExpressionType +from .numeric_expr import NumericExpression import operator @@ -240,6 +239,26 @@ def atleast(n, *args): return result +def all_different(*args): + """Creates a new AllDifferentExpression + + Requires all of the arguments to take on a different value + + Usage: all_different(m.X1, m.X2, ...) + """ + return AllDifferentExpression(list(_flattened(args))) + + +def count_if(*args): + """Creates a new CountIfExpression + + Counts the number of True-valued arguments + + Usage: count_if(m.Y1, m.Y2, ...) + """ + return CountIfExpression(list(_flattened(args))) + + class UnaryBooleanExpression(BooleanExpression): """ Abstract class for single-argument logical expressions. @@ -512,4 +531,52 @@ def _apply_operation(self, result): return sum(result[1:]) >= result[0] +class AllDifferentExpression(NaryBooleanExpression): + """ + Logical expression that all of the N child statements have different values. + All arguments are expected to be discrete-valued. + """ + __slots__ = () + + PRECEDENCE = 9 # TODO: maybe? + + def getname(self, *arg, **kwd): + return 'all_different' + + def _to_string(self, values, verbose, smap): + return "all_different(%s)" % (", ".join(values)) + + def _apply_operation(self, result): + for val1, val2 in combinations(result, 2): + if val1 == val2: + return False + return True + +class CountIfExpression(NumericExpression): + """ + Logical expression that returns the number of True child statements. + All arguments are expected to be Boolean-valued. + """ + __slots__ = () + PRECEDENCE = 10 # TODO: maybe? + + def __init__(self, args): + # require a list, a la SumExpression + if args.__class__ is not list: + args = list(args) + self._args_ = args + + # NumericExpression assumes binary operator, so we have to override. + def nargs(self): + return len(self._args_) + + def getname(self, *arg, **kwd): + return 'count_if' + + def _to_string(self, values, verbose, smap): + return "count_if(%s)" % (", ".join(values)) + + def _apply_operation(self, result): + return sum(r for r in result) + special_boolean_atom_types = {ExactlyExpression, AtMostExpression, AtLeastExpression} diff --git a/pyomo/core/tests/unit/test_logical_expr_expanded.py b/pyomo/core/tests/unit/test_logical_expr_expanded.py index f5b86d59cbd..d494c13c83c 100644 --- a/pyomo/core/tests/unit/test_logical_expr_expanded.py +++ b/pyomo/core/tests/unit/test_logical_expr_expanded.py @@ -15,7 +15,7 @@ """ from __future__ import division import operator -from itertools import product +from itertools import permutations, product import pyomo.common.unittest as unittest @@ -23,6 +23,8 @@ from pyomo.core.expr.sympy_tools import sympy_available from pyomo.core.expr.visitor import identify_variables from pyomo.environ import ( + all_different, + count_if, land, atleast, atmost, @@ -39,6 +41,8 @@ BooleanVar, lnot, xor, + Var, + Integers ) @@ -234,6 +238,42 @@ def test_nary_atleast(self): ) self.assertEqual(value(atleast(ntrue, m.Y)), correct_value) + def test_nary_all_diff(self): + m = ConcreteModel() + m.x = Var(range(4), domain=Integers, bounds=(0, 3)) + for vals in permutations(range(4)): + self.assertTrue(value(all_different(*vals))) + for i, v in enumerate(vals): + m.x[i] = v + self.assertTrue(value(all_different(m.x))) + self.assertFalse(value(all_different(1, 1, 2, 3))) + m.x[0] = 1 + m.x[1] = 1 + m.x[2] = 2 + m.x[3] = 3 + self.assertFalse(value(all_different(m.x))) + + def test_count_if(self): + nargs = 3 + m = ConcreteModel() + m.s = RangeSet(nargs) + m.Y = BooleanVar(m.s) + m.x = Var(domain=Integers, bounds=(0, 3)) + for truth_combination in _generate_possible_truth_inputs(nargs): + for ntrue in range(nargs + 1): + m.Y.set_values(dict(enumerate(truth_combination, 1))) + correct_value = sum(truth_combination) + self.assertEqual( + value(count_if(*(m.Y[i] for i in m.s))), correct_value + ) + self.assertEqual(value(count_if(m.Y)), correct_value) + m.x = 2 + self.assertEqual(value(count_if([m.Y[i] for i in m.s] + [m.x == 3,])), + correct_value) + m.x = 3 + self.assertEqual(value(count_if([m.Y[i] for i in m.s] + [m.x == 3,])), + correct_value + 1) + def test_to_string(self): m = ConcreteModel() m.Y1 = BooleanVar() @@ -249,6 +289,8 @@ def test_to_string(self): self.assertEqual(str(atleast(1, m.Y1, m.Y2)), "atleast(1: [Y1, Y2])") self.assertEqual(str(atmost(1, m.Y1, m.Y2)), "atmost(1: [Y1, Y2])") self.assertEqual(str(exactly(1, m.Y1, m.Y2)), "exactly(1: [Y1, Y2])") + self.assertEqual(str(all_different(m.Y1, m.Y2)), "all_different(Y1, Y2)") + self.assertEqual(str(count_if(m.Y1, m.Y2)), "count_if(Y1, Y2)") # Precedence checks self.assertEqual(str(m.Y1.implies(m.Y2).lor(m.Y3)), "(Y1 --> Y2) ∨ Y3") @@ -271,6 +313,8 @@ def test_node_types(self): self.assertTrue(lnot(m.Y1).is_expression_type()) self.assertTrue(equivalent(m.Y1, m.Y2).is_expression_type()) self.assertTrue(atmost(1, [m.Y1, m.Y2, m.Y3]).is_expression_type()) + self.assertTrue(all_different(m.Y1, m.Y2, m.Y3).is_expression_type()) + self.assertTrue(count_if(m.Y1, m.Y2, m.Y3).is_expression_type()) def test_numeric_invalid(self): m = ConcreteModel() diff --git a/pyomo/environ/__init__.py b/pyomo/environ/__init__.py index 51c68449247..c3fb3ec4a85 100644 --- a/pyomo/environ/__init__.py +++ b/pyomo/environ/__init__.py @@ -114,6 +114,8 @@ def _import_packages(): exactly, atleast, atmost, + all_different, + count_if, implies, lnot, xor, From b3bc17bc493a5b644543847a938d94a5b19bc7ee Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 20 Nov 2023 16:44:58 -0700 Subject: [PATCH 017/103] black --- pyomo/core/expr/logical_expr.py | 8 ++++++-- .../tests/unit/test_logical_expr_expanded.py | 16 ++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pyomo/core/expr/logical_expr.py b/pyomo/core/expr/logical_expr.py index 2b261278ee9..45f6cfaf7eb 100644 --- a/pyomo/core/expr/logical_expr.py +++ b/pyomo/core/expr/logical_expr.py @@ -536,9 +536,10 @@ class AllDifferentExpression(NaryBooleanExpression): Logical expression that all of the N child statements have different values. All arguments are expected to be discrete-valued. """ + __slots__ = () - PRECEDENCE = 9 # TODO: maybe? + PRECEDENCE = 9 # TODO: maybe? def getname(self, *arg, **kwd): return 'all_different' @@ -552,13 +553,15 @@ def _apply_operation(self, result): return False return True + class CountIfExpression(NumericExpression): """ Logical expression that returns the number of True child statements. All arguments are expected to be Boolean-valued. """ + __slots__ = () - PRECEDENCE = 10 # TODO: maybe? + PRECEDENCE = 10 # TODO: maybe? def __init__(self, args): # require a list, a la SumExpression @@ -579,4 +582,5 @@ def _to_string(self, values, verbose, smap): def _apply_operation(self, result): return sum(r for r in result) + special_boolean_atom_types = {ExactlyExpression, AtMostExpression, AtLeastExpression} diff --git a/pyomo/core/tests/unit/test_logical_expr_expanded.py b/pyomo/core/tests/unit/test_logical_expr_expanded.py index d494c13c83c..9e68fee441f 100644 --- a/pyomo/core/tests/unit/test_logical_expr_expanded.py +++ b/pyomo/core/tests/unit/test_logical_expr_expanded.py @@ -42,7 +42,7 @@ lnot, xor, Var, - Integers + Integers, ) @@ -263,16 +263,16 @@ def test_count_if(self): for ntrue in range(nargs + 1): m.Y.set_values(dict(enumerate(truth_combination, 1))) correct_value = sum(truth_combination) - self.assertEqual( - value(count_if(*(m.Y[i] for i in m.s))), correct_value - ) + self.assertEqual(value(count_if(*(m.Y[i] for i in m.s))), correct_value) self.assertEqual(value(count_if(m.Y)), correct_value) m.x = 2 - self.assertEqual(value(count_if([m.Y[i] for i in m.s] + [m.x == 3,])), - correct_value) + self.assertEqual( + value(count_if([m.Y[i] for i in m.s] + [m.x == 3])), correct_value + ) m.x = 3 - self.assertEqual(value(count_if([m.Y[i] for i in m.s] + [m.x == 3,])), - correct_value + 1) + self.assertEqual( + value(count_if([m.Y[i] for i in m.s] + [m.x == 3])), correct_value + 1 + ) def test_to_string(self): m = ConcreteModel() From f261fdc146d1de7b75c13d7fbc2d8a3e97a0fe7f Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Tue, 21 Nov 2023 15:19:10 -0500 Subject: [PATCH 018/103] fix load_solutions bug --- pyomo/contrib/mindtpy/algorithm_base_class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index d485dc4651f..4ea492bd7c9 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -864,7 +864,7 @@ def init_rNLP(self, add_oa_cuts=True): results = self.nlp_opt.solve( self.rnlp, tee=config.nlp_solver_tee, - load_solutions=config.load_solutions, + load_solutions=self.load_solutions, **nlp_args, ) if len(results.solution) > 0: From a6802a53882df831a8e7c0c91b18f972c608d385 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 21 Nov 2023 14:58:57 -0700 Subject: [PATCH 019/103] Adding all_different and count_if to docplex writer, testing them, fixing a bug with evaluation of count_if --- pyomo/contrib/cp/repn/docplex_writer.py | 19 ++- pyomo/contrib/cp/tests/test_docplex_walker.py | 58 ++++++++- pyomo/contrib/cp/tests/test_docplex_writer.py | 116 ++++++++++++++++++ pyomo/core/expr/__init__.py | 2 + pyomo/core/expr/logical_expr.py | 2 +- 5 files changed, 194 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/cp/repn/docplex_writer.py b/pyomo/contrib/cp/repn/docplex_writer.py index 51c3f66140e..c5b219ae9fd 100644 --- a/pyomo/contrib/cp/repn/docplex_writer.py +++ b/pyomo/contrib/cp/repn/docplex_writer.py @@ -64,7 +64,7 @@ IndexedBooleanVar, ) from pyomo.core.base.expression import ScalarExpression, _GeneralExpressionData -from pyomo.core.base.param import IndexedParam, ScalarParam +from pyomo.core.base.param import IndexedParam, ScalarParam, _ParamData from pyomo.core.base.var import ScalarVar, _GeneralVarData, IndexedVar import pyomo.core.expr as EXPR from pyomo.core.expr.visitor import StreamBasedExpressionVisitor, identify_variables @@ -805,6 +805,20 @@ def _handle_at_least_node(visitor, node, *args): ) +def _handle_all_diff_node(visitor, node, *args): + return ( + _GENERAL, + cp.all_diff(_get_int_valued_expr(arg) for arg in args), + ) + + +def _handle_count_if_node(visitor, node, *args): + return ( + _GENERAL, + cp.count((_get_bool_valued_expr(arg) for arg in args), 1), + ) + + ## CallExpression handllers @@ -932,6 +946,8 @@ class LogicalToDoCplex(StreamBasedExpressionVisitor): EXPR.ExactlyExpression: _handle_exactly_node, EXPR.AtMostExpression: _handle_at_most_node, EXPR.AtLeastExpression: _handle_at_least_node, + EXPR.AllDifferentExpression: _handle_all_diff_node, + EXPR.CountIfExpression: _handle_count_if_node, EXPR.EqualityExpression: _handle_equality_node, EXPR.NotEqualExpression: _handle_not_equal_node, EXPR.InequalityExpression: _handle_inequality_node, @@ -960,6 +976,7 @@ class LogicalToDoCplex(StreamBasedExpressionVisitor): ScalarExpression: _before_named_expression, IndexedParam: _before_indexed_param, # Because of indirection ScalarParam: _before_param, + _ParamData: _before_param, } def __init__(self, cpx_model, symbolic_solver_labels=False): diff --git a/pyomo/contrib/cp/tests/test_docplex_walker.py b/pyomo/contrib/cp/tests/test_docplex_walker.py index 97bc538c827..f8c5f7f766f 100644 --- a/pyomo/contrib/cp/tests/test_docplex_walker.py +++ b/pyomo/contrib/cp/tests/test_docplex_walker.py @@ -21,7 +21,9 @@ from pyomo.core.base.range import NumericRange from pyomo.core.expr.numeric_expr import MinExpression, MaxExpression -from pyomo.core.expr.logical_expr import equivalent, exactly, atleast, atmost +from pyomo.core.expr.logical_expr import ( + equivalent, exactly, atleast, atmost, all_different, count_if +) from pyomo.core.expr.relational_expr import NotEqualExpression from pyomo.environ import ( @@ -401,6 +403,60 @@ def test_atmost_expression(self): expr[1].equals(cp.less_or_equal(cp.count([a[i] == 4 for i in m.I], 1), 3)) ) + def test_all_diff_expression(self): + m = self.get_model() + m.a.domain = Integers + m.a.bounds = (11, 20) + m.c = LogicalConstraint(expr=all_different(m.a)) + + visitor = self.get_visitor() + expr = visitor.walk_expression((m.c.body, m.c, 0)) + + a = {} + for i in m.I: + self.assertIn(id(m.a[i]), visitor.var_map) + a[i] = visitor.var_map[id(m.a[i])] + + self.assertTrue( + expr[1].equals(cp.all_diff(a[i] for i in m.I)) + ) + + def test_Boolean_args_in_all_diff_expression(self): + m = self.get_model() + m.a.domain = Integers + m.a.bounds = (11, 20) + m.c = LogicalConstraint(expr=all_different(m.a[1] == 13, m.b)) + + visitor = self.get_visitor() + expr = visitor.walk_expression((m.c.body, m.c, 0)) + + self.assertIn(id(m.a[1]), visitor.var_map) + a0 = visitor.var_map[id(m.a[1])] + self.assertIn(id(m.b), visitor.var_map) + b = visitor.var_map[id(m.b)] + + self.assertTrue( + expr[1].equals(cp.all_diff(a0 == 13, b)) + ) + + def test_count_if_expression(self): + m = self.get_model() + m.a.domain = Integers + m.a.bounds = (11, 20) + m.c = Constraint(expr=count_if(m.a[i] == i for i in m.I) == 5) + + visitor = self.get_visitor() + expr = visitor.walk_expression((m.c.expr, m.c, 0)) + + a = {} + for i in m.I: + self.assertIn(id(m.a[i]), visitor.var_map) + a[i] = visitor.var_map[id(m.a[i])] + + self.assertTrue( + expr[1].equals(cp.count((a[i] == i for i in m.I), 1) == 5) + ) + def test_interval_var_is_present(self): m = self.get_model() m.a.domain = Integers diff --git a/pyomo/contrib/cp/tests/test_docplex_writer.py b/pyomo/contrib/cp/tests/test_docplex_writer.py index d569ef2e696..566b4084daa 100644 --- a/pyomo/contrib/cp/tests/test_docplex_writer.py +++ b/pyomo/contrib/cp/tests/test_docplex_writer.py @@ -15,10 +15,13 @@ from pyomo.contrib.cp import IntervalVar, Pulse, Step, AlwaysIn from pyomo.contrib.cp.repn.docplex_writer import LogicalToDoCplex from pyomo.environ import ( + all_different, + count_if, ConcreteModel, Set, Var, Integers, + Param, LogicalConstraint, implies, value, @@ -254,3 +257,116 @@ def x_bounds(m, i): self.assertEqual(results.problem.sense, minimize) self.assertEqual(results.problem.lower_bound, 6) self.assertEqual(results.problem.upper_bound, 6) + + def test_matching_problem(self): + m = ConcreteModel() + + m.People = Set(initialize=['P1', 'P2', 'P3', 'P4', 'P5', 'P6', 'P7']) + m.Languages = Set(initialize=['English', 'Spanish', 'Hindi', 'Swedish']) + # People have integer names because we don't have categorical vars yet. + m.Names = Set(initialize=range(len(m.People))) + + m.Observed = Param(m.Names, m.Names, m.Languages, + initialize={ + (0, 1, 'English'): 1, + (1, 0, 'English'): 1, + (0, 2, 'English'): 1, + (2, 0, 'English'): 1, + (0, 3, 'English'): 1, + (3, 0, 'English'): 1, + (0, 4, 'English'): 1, + (4, 0, 'English'): 1, + (0, 5, 'English'): 1, + (5, 0, 'English'): 1, + (0, 6, 'English'): 1, + (6, 0, 'English'): 1, + (1, 2, 'Spanish'): 1, + (2, 1, 'Spanish'): 1, + (1, 5, 'Hindi'): 1, + (5, 1, 'Hindi'): 1, + (1, 6, 'Hindi'): 1, + (6, 1, 'Hindi'): 1, + (2, 3, 'Swedish'): 1, + (3, 2, 'Swedish'): 1, + (3, 4, 'English'): 1, + (4, 3, 'English'): 1, + }, default=0, mutable=True)# TODO: shouldn't need to + # be mutable, but waiting + # on #3045 + + m.Expected = Param(m.People, m.People, m.Languages, initialize={ + ('P1', 'P2', 'English') : 1, + ('P2', 'P1', 'English') : 1, + ('P1', 'P3', 'English') : 1, + ('P3', 'P1', 'English') : 1, + ('P1', 'P4', 'English') : 1, + ('P4', 'P1', 'English') : 1, + ('P1', 'P5', 'English') : 1, + ('P5', 'P1', 'English') : 1, + ('P1', 'P6', 'English') : 1, + ('P6', 'P1', 'English') : 1, + ('P1', 'P7', 'English') : 1, + ('P7', 'P1', 'English') : 1, + ('P2', 'P3', 'Spanish') : 1, + ('P3', 'P2', 'Spanish') : 1, + ('P2', 'P6', 'Hindi') : 1, + ('P6', 'P2', 'Hindi') : 1, + ('P2', 'P7', 'Hindi') : 1, + ('P7', 'P2', 'Hindi') : 1, + ('P3', 'P4', 'Swedish') : 1, + ('P4', 'P3', 'Swedish') : 1, + ('P4', 'P5', 'English') : 1, + ('P5', 'P4', 'English') : 1, + }, default=0, mutable=True)# TODO: shouldn't need to be mutable, but + # waiting on #3045 + + m.person_name = Var(m.People, bounds=(0, max(m.Names)), domain=Integers) + + m.one_to_one = LogicalConstraint(expr=all_different(m.person_name[person] for + person in m.People)) + + + m.obj = Objective(expr=count_if(m.Observed[m.person_name[p1], + m.person_name[p2], l] == + m.Expected[p1, p2, l] for p1 + in m.People for p2 in + m.People for l in + m.Languages), sense=maximize) + + results = SolverFactory('cp_optimizer').solve(m) + + # we can get one of two perfect matches: + perfect = 7*7*4 + self.assertEqual(results.problem.lower_bound, perfect) + self.assertEqual(results.problem.upper_bound, perfect) + self.assertEqual(results.solver.termination_condition, + TerminationCondition.optimal) + self.assertEqual(value(m.obj), perfect) + m.person_name.pprint() + self.assertEqual(value(m.person_name['P1']), 0) + self.assertEqual(value(m.person_name['P2']), 1) + self.assertEqual(value(m.person_name['P3']), 2) + self.assertEqual(value(m.person_name['P4']), 3) + self.assertEqual(value(m.person_name['P5']), 4) + # We can't distinguish P6 and P7, so they could each have either of + # names 5 and 6 + self.assertTrue(value(m.person_name['P6']) == 5 or + value(m.person_name['P6']) == 6) + self.assertTrue(value(m.person_name['P7']) == 5 or + value(m.person_name['P7']) == 6) + + m.person_name['P6'].fix(5) + m.person_name['P7'].fix(6) + + results = SolverFactory('cp_optimizer').solve(m) + self.assertEqual(results.solver.termination_condition, + TerminationCondition.optimal) + self.assertEqual(value(m.obj), perfect) + + m.person_name['P6'].fix(6) + m.person_name['P7'].fix(5) + + results = SolverFactory('cp_optimizer').solve(m) + self.assertEqual(results.solver.termination_condition, + TerminationCondition.optimal) + self.assertEqual(value(m.obj), perfect) diff --git a/pyomo/core/expr/__init__.py b/pyomo/core/expr/__init__.py index de2228189f9..bd6d1b995a1 100644 --- a/pyomo/core/expr/__init__.py +++ b/pyomo/core/expr/__init__.py @@ -70,6 +70,8 @@ ExactlyExpression, AtMostExpression, AtLeastExpression, + AllDifferentExpression, + CountIfExpression, # land, lnot, diff --git a/pyomo/core/expr/logical_expr.py b/pyomo/core/expr/logical_expr.py index 45f6cfaf7eb..31082293a71 100644 --- a/pyomo/core/expr/logical_expr.py +++ b/pyomo/core/expr/logical_expr.py @@ -580,7 +580,7 @@ def _to_string(self, values, verbose, smap): return "count_if(%s)" % (", ".join(values)) def _apply_operation(self, result): - return sum(r for r in result) + return sum(value(r) for r in result) special_boolean_atom_types = {ExactlyExpression, AtMostExpression, AtLeastExpression} From f70001b63198b6457c63e717b9b37933c999acfc Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 21 Nov 2023 14:59:32 -0700 Subject: [PATCH 020/103] Blackify --- pyomo/contrib/cp/repn/docplex_writer.py | 10 +- pyomo/contrib/cp/tests/test_docplex_walker.py | 19 +- pyomo/contrib/cp/tests/test_docplex_writer.py | 168 ++++++++++-------- 3 files changed, 106 insertions(+), 91 deletions(-) diff --git a/pyomo/contrib/cp/repn/docplex_writer.py b/pyomo/contrib/cp/repn/docplex_writer.py index c5b219ae9fd..c2687662fe8 100644 --- a/pyomo/contrib/cp/repn/docplex_writer.py +++ b/pyomo/contrib/cp/repn/docplex_writer.py @@ -806,17 +806,11 @@ def _handle_at_least_node(visitor, node, *args): def _handle_all_diff_node(visitor, node, *args): - return ( - _GENERAL, - cp.all_diff(_get_int_valued_expr(arg) for arg in args), - ) + return (_GENERAL, cp.all_diff(_get_int_valued_expr(arg) for arg in args)) def _handle_count_if_node(visitor, node, *args): - return ( - _GENERAL, - cp.count((_get_bool_valued_expr(arg) for arg in args), 1), - ) + return (_GENERAL, cp.count((_get_bool_valued_expr(arg) for arg in args), 1)) ## CallExpression handllers diff --git a/pyomo/contrib/cp/tests/test_docplex_walker.py b/pyomo/contrib/cp/tests/test_docplex_walker.py index f8c5f7f766f..0f1c73cd3b1 100644 --- a/pyomo/contrib/cp/tests/test_docplex_walker.py +++ b/pyomo/contrib/cp/tests/test_docplex_walker.py @@ -22,7 +22,12 @@ from pyomo.core.base.range import NumericRange from pyomo.core.expr.numeric_expr import MinExpression, MaxExpression from pyomo.core.expr.logical_expr import ( - equivalent, exactly, atleast, atmost, all_different, count_if + equivalent, + exactly, + atleast, + atmost, + all_different, + count_if, ) from pyomo.core.expr.relational_expr import NotEqualExpression @@ -417,9 +422,7 @@ def test_all_diff_expression(self): self.assertIn(id(m.a[i]), visitor.var_map) a[i] = visitor.var_map[id(m.a[i])] - self.assertTrue( - expr[1].equals(cp.all_diff(a[i] for i in m.I)) - ) + self.assertTrue(expr[1].equals(cp.all_diff(a[i] for i in m.I))) def test_Boolean_args_in_all_diff_expression(self): m = self.get_model() @@ -435,9 +438,7 @@ def test_Boolean_args_in_all_diff_expression(self): self.assertIn(id(m.b), visitor.var_map) b = visitor.var_map[id(m.b)] - self.assertTrue( - expr[1].equals(cp.all_diff(a0 == 13, b)) - ) + self.assertTrue(expr[1].equals(cp.all_diff(a0 == 13, b))) def test_count_if_expression(self): m = self.get_model() @@ -453,9 +454,7 @@ def test_count_if_expression(self): self.assertIn(id(m.a[i]), visitor.var_map) a[i] = visitor.var_map[id(m.a[i])] - self.assertTrue( - expr[1].equals(cp.count((a[i] == i for i in m.I), 1) == 5) - ) + self.assertTrue(expr[1].equals(cp.count((a[i] == i for i in m.I), 1) == 5)) def test_interval_var_is_present(self): m = self.get_model() diff --git a/pyomo/contrib/cp/tests/test_docplex_writer.py b/pyomo/contrib/cp/tests/test_docplex_writer.py index 566b4084daa..b563052ef3a 100644 --- a/pyomo/contrib/cp/tests/test_docplex_writer.py +++ b/pyomo/contrib/cp/tests/test_docplex_writer.py @@ -266,81 +266,99 @@ def test_matching_problem(self): # People have integer names because we don't have categorical vars yet. m.Names = Set(initialize=range(len(m.People))) - m.Observed = Param(m.Names, m.Names, m.Languages, - initialize={ - (0, 1, 'English'): 1, - (1, 0, 'English'): 1, - (0, 2, 'English'): 1, - (2, 0, 'English'): 1, - (0, 3, 'English'): 1, - (3, 0, 'English'): 1, - (0, 4, 'English'): 1, - (4, 0, 'English'): 1, - (0, 5, 'English'): 1, - (5, 0, 'English'): 1, - (0, 6, 'English'): 1, - (6, 0, 'English'): 1, - (1, 2, 'Spanish'): 1, - (2, 1, 'Spanish'): 1, - (1, 5, 'Hindi'): 1, - (5, 1, 'Hindi'): 1, - (1, 6, 'Hindi'): 1, - (6, 1, 'Hindi'): 1, - (2, 3, 'Swedish'): 1, - (3, 2, 'Swedish'): 1, - (3, 4, 'English'): 1, - (4, 3, 'English'): 1, - }, default=0, mutable=True)# TODO: shouldn't need to - # be mutable, but waiting - # on #3045 - - m.Expected = Param(m.People, m.People, m.Languages, initialize={ - ('P1', 'P2', 'English') : 1, - ('P2', 'P1', 'English') : 1, - ('P1', 'P3', 'English') : 1, - ('P3', 'P1', 'English') : 1, - ('P1', 'P4', 'English') : 1, - ('P4', 'P1', 'English') : 1, - ('P1', 'P5', 'English') : 1, - ('P5', 'P1', 'English') : 1, - ('P1', 'P6', 'English') : 1, - ('P6', 'P1', 'English') : 1, - ('P1', 'P7', 'English') : 1, - ('P7', 'P1', 'English') : 1, - ('P2', 'P3', 'Spanish') : 1, - ('P3', 'P2', 'Spanish') : 1, - ('P2', 'P6', 'Hindi') : 1, - ('P6', 'P2', 'Hindi') : 1, - ('P2', 'P7', 'Hindi') : 1, - ('P7', 'P2', 'Hindi') : 1, - ('P3', 'P4', 'Swedish') : 1, - ('P4', 'P3', 'Swedish') : 1, - ('P4', 'P5', 'English') : 1, - ('P5', 'P4', 'English') : 1, - }, default=0, mutable=True)# TODO: shouldn't need to be mutable, but - # waiting on #3045 + m.Observed = Param( + m.Names, + m.Names, + m.Languages, + initialize={ + (0, 1, 'English'): 1, + (1, 0, 'English'): 1, + (0, 2, 'English'): 1, + (2, 0, 'English'): 1, + (0, 3, 'English'): 1, + (3, 0, 'English'): 1, + (0, 4, 'English'): 1, + (4, 0, 'English'): 1, + (0, 5, 'English'): 1, + (5, 0, 'English'): 1, + (0, 6, 'English'): 1, + (6, 0, 'English'): 1, + (1, 2, 'Spanish'): 1, + (2, 1, 'Spanish'): 1, + (1, 5, 'Hindi'): 1, + (5, 1, 'Hindi'): 1, + (1, 6, 'Hindi'): 1, + (6, 1, 'Hindi'): 1, + (2, 3, 'Swedish'): 1, + (3, 2, 'Swedish'): 1, + (3, 4, 'English'): 1, + (4, 3, 'English'): 1, + }, + default=0, + mutable=True, + ) # TODO: shouldn't need to + # be mutable, but waiting + # on #3045 + + m.Expected = Param( + m.People, + m.People, + m.Languages, + initialize={ + ('P1', 'P2', 'English'): 1, + ('P2', 'P1', 'English'): 1, + ('P1', 'P3', 'English'): 1, + ('P3', 'P1', 'English'): 1, + ('P1', 'P4', 'English'): 1, + ('P4', 'P1', 'English'): 1, + ('P1', 'P5', 'English'): 1, + ('P5', 'P1', 'English'): 1, + ('P1', 'P6', 'English'): 1, + ('P6', 'P1', 'English'): 1, + ('P1', 'P7', 'English'): 1, + ('P7', 'P1', 'English'): 1, + ('P2', 'P3', 'Spanish'): 1, + ('P3', 'P2', 'Spanish'): 1, + ('P2', 'P6', 'Hindi'): 1, + ('P6', 'P2', 'Hindi'): 1, + ('P2', 'P7', 'Hindi'): 1, + ('P7', 'P2', 'Hindi'): 1, + ('P3', 'P4', 'Swedish'): 1, + ('P4', 'P3', 'Swedish'): 1, + ('P4', 'P5', 'English'): 1, + ('P5', 'P4', 'English'): 1, + }, + default=0, + mutable=True, + ) # TODO: shouldn't need to be mutable, but + # waiting on #3045 m.person_name = Var(m.People, bounds=(0, max(m.Names)), domain=Integers) - m.one_to_one = LogicalConstraint(expr=all_different(m.person_name[person] for - person in m.People)) - + m.one_to_one = LogicalConstraint( + expr=all_different(m.person_name[person] for person in m.People) + ) - m.obj = Objective(expr=count_if(m.Observed[m.person_name[p1], - m.person_name[p2], l] == - m.Expected[p1, p2, l] for p1 - in m.People for p2 in - m.People for l in - m.Languages), sense=maximize) + m.obj = Objective( + expr=count_if( + m.Observed[m.person_name[p1], m.person_name[p2], l] + == m.Expected[p1, p2, l] + for p1 in m.People + for p2 in m.People + for l in m.Languages + ), + sense=maximize, + ) results = SolverFactory('cp_optimizer').solve(m) # we can get one of two perfect matches: - perfect = 7*7*4 + perfect = 7 * 7 * 4 self.assertEqual(results.problem.lower_bound, perfect) self.assertEqual(results.problem.upper_bound, perfect) - self.assertEqual(results.solver.termination_condition, - TerminationCondition.optimal) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) self.assertEqual(value(m.obj), perfect) m.person_name.pprint() self.assertEqual(value(m.person_name['P1']), 0) @@ -350,23 +368,27 @@ def test_matching_problem(self): self.assertEqual(value(m.person_name['P5']), 4) # We can't distinguish P6 and P7, so they could each have either of # names 5 and 6 - self.assertTrue(value(m.person_name['P6']) == 5 or - value(m.person_name['P6']) == 6) - self.assertTrue(value(m.person_name['P7']) == 5 or - value(m.person_name['P7']) == 6) + self.assertTrue( + value(m.person_name['P6']) == 5 or value(m.person_name['P6']) == 6 + ) + self.assertTrue( + value(m.person_name['P7']) == 5 or value(m.person_name['P7']) == 6 + ) m.person_name['P6'].fix(5) m.person_name['P7'].fix(6) results = SolverFactory('cp_optimizer').solve(m) - self.assertEqual(results.solver.termination_condition, - TerminationCondition.optimal) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) self.assertEqual(value(m.obj), perfect) m.person_name['P6'].fix(6) m.person_name['P7'].fix(5) results = SolverFactory('cp_optimizer').solve(m) - self.assertEqual(results.solver.termination_condition, - TerminationCondition.optimal) + self.assertEqual( + results.solver.termination_condition, TerminationCondition.optimal + ) self.assertEqual(value(m.obj), perfect) From 015ebaf8593d0f21336af323ac8d0d440e17c9f4 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Mon, 27 Nov 2023 14:41:36 -0500 Subject: [PATCH 021/103] fix typo: change try to trying --- pyomo/contrib/mindtpy/algorithm_base_class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index 4ea492bd7c9..a7a8a41cd70 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -857,7 +857,7 @@ def init_rNLP(self, add_oa_cuts=True): not in self.mip_objective_polynomial_degree ): config.logger.info( - 'Initial relaxed NLP problem is infeasible. This might be related to partition_obj_nonlinear_terms. Try to solve it again without partitioning nonlinear objective function.' + 'Initial relaxed NLP problem is infeasible. This might be related to partition_obj_nonlinear_terms. Trying to solve it again without partitioning nonlinear objective function.' ) self.rnlp.MindtPy_utils.objective.deactivate() self.rnlp.MindtPy_utils.objective_list[0].activate() From a6079d50b319cc0288ee8612bf58e2f02a88fe09 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Mon, 27 Nov 2023 15:37:57 -0500 Subject: [PATCH 022/103] add more details of the error in copy_var_list_values --- pyomo/contrib/mindtpy/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 48c8aab31c4..ea2136b0589 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -1010,4 +1010,5 @@ def copy_var_list_values( elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: v_to.set_value(0) else: - raise ValueError("copy_var_list_values failed.") + raise ValueError("copy_var_list_values failed with variable {}, value = {} and rounded value = {}" + "".format(v_to.name, var_val, rounded_val)) From e8b3b72df0d5c869be0a169ea5310940da342049 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Mon, 27 Nov 2023 18:26:12 -0500 Subject: [PATCH 023/103] create copy_var_value function --- pyomo/contrib/mindtpy/algorithm_base_class.py | 1 - pyomo/contrib/mindtpy/single_tree.py | 53 +------- pyomo/contrib/mindtpy/tests/test_mindtpy.py | 1 + pyomo/contrib/mindtpy/util.py | 121 ++++++++++-------- 4 files changed, 74 insertions(+), 102 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index a7a8a41cd70..92e1075fe90 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -1853,7 +1853,6 @@ def handle_main_optimal(self, main_mip, update_bound=True): f"Integer variable {var.name} not initialized. " "Setting it to its lower bound" ) - # nlp_var.bounds[0] var.set_value(var.lb, skip_validation=True) # warm start for the nlp subproblem copy_var_list_values( diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 66435c2587f..a5d4401d623 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -16,12 +16,11 @@ from pyomo.repn import generate_standard_repn import pyomo.core.expr as EXPR from math import copysign -from pyomo.contrib.mindtpy.util import get_integer_solution, copy_var_list_values +from pyomo.contrib.mindtpy.util import get_integer_solution, copy_var_list_values, copy_var_value from pyomo.contrib.gdpopt.util import get_main_elapsed_time, time_code from pyomo.opt import TerminationCondition as tc from pyomo.core import minimize, value from pyomo.core.expr import identify_variables -import math cplex, cplex_available = attempt_import('cplex') @@ -35,7 +34,6 @@ def copy_lazy_var_list_values( self, opt, from_list, to_list, config, skip_stale=False, skip_fixed=True ): """This function copies variable values from one list to another. - Rounds to Binary/Integer if necessary. Sets to zero for NonNegativeReals if necessary. @@ -44,17 +42,15 @@ def copy_lazy_var_list_values( opt : SolverFactory The cplex_persistent solver. from_list : list - The variables that provides the values to copy from. + The variable list that provides the values to copy from. to_list : list - The variables that need to set value. + The variable list that needs to set value. config : ConfigBlock The specific configurations for MindtPy. skip_stale : bool, optional Whether to skip the stale variables, by default False. skip_fixed : bool, optional Whether to skip the fixed variables, by default True. - ignore_integrality : bool, optional - Whether to ignore the integrality of integer variables, by default False. """ for v_from, v_to in zip(from_list, to_list): if skip_stale and v_from.stale: @@ -62,48 +58,7 @@ def copy_lazy_var_list_values( if skip_fixed and v_to.is_fixed(): continue # Skip fixed variables. v_val = self.get_values(opt._pyomo_var_to_solver_var_map[v_from]) - rounded_val = int(round(v_val)) - # We don't want to trigger the reset of the global stale - # indicator, so we will set this variable to be "stale", - # knowing that set_value will switch it back to "not - # stale" - v_to.stale = True - # NOTE: PEP 2180 changes the var behavior so that domain - # / bounds violations no longer generate exceptions (and - # instead log warnings). This means that the following - # will always succeed and the ValueError should never be - # raised. - if ( - v_val in v_to.domain - and not ((v_to.has_lb() and v_val < v_to.lb)) - and not ((v_to.has_ub() and v_val > v_to.ub)) - ): - v_to.set_value(v_val) - # Snap the value to the bounds - # TODO: check the performance of - # v_to.lb - v_val <= config.variable_tolerance - elif ( - v_to.has_lb() - and v_val < v_to.lb - # and v_to.lb - v_val <= config.variable_tolerance - ): - v_to.set_value(v_to.lb) - elif ( - v_to.has_ub() - and v_val > v_to.ub - # and v_val - v_to.ub <= config.variable_tolerance - ): - v_to.set_value(v_to.ub) - # ... or the nearest integer - elif ( - v_to.is_integer() - and math.fabs(v_val - rounded_val) <= config.integer_tolerance - ): # and rounded_val in v_to.domain: - v_to.set_value(rounded_val) - elif abs(v_val) <= config.zero_tolerance and 0 in v_to.domain: - v_to.set_value(0) - else: - raise ValueError('copy_lazy_var_list_values failed.') + copy_var_value(v_from, v_to, v_val, config, ignore_integrality=False) def add_lazy_oa_cuts( self, diff --git a/pyomo/contrib/mindtpy/tests/test_mindtpy.py b/pyomo/contrib/mindtpy/tests/test_mindtpy.py index e872eccc670..ae531f9bd84 100644 --- a/pyomo/contrib/mindtpy/tests/test_mindtpy.py +++ b/pyomo/contrib/mindtpy/tests/test_mindtpy.py @@ -327,6 +327,7 @@ def test_OA_APPSI_ipopt(self): value(model.objective.expr), model.optimal_value, places=1 ) + # CYIPOPT will raise WARNING (W1002) during loading solution. @unittest.skipUnless( SolverFactory('cyipopt').available(exception_flag=False), "APPSI_IPOPT not available.", diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index ea2136b0589..2970a805540 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -693,37 +693,7 @@ def copy_var_list_values_from_solution_pool( elif config.mip_solver == 'gurobi_persistent': solver_model.setParam(gurobipy.GRB.Param.SolutionNumber, solution_name) var_val = var_map[v_from].Xn - # We don't want to trigger the reset of the global stale - # indicator, so we will set this variable to be "stale", - # knowing that set_value will switch it back to "not - # stale" - v_to.stale = True - rounded_val = int(round(var_val)) - # NOTE: PEP 2180 changes the var behavior so that domain / - # bounds violations no longer generate exceptions (and - # instead log warnings). This means that the following will - # always succeed and the ValueError should never be raised. - if ( - var_val in v_to.domain - and not ((v_to.has_lb() and var_val < v_to.lb)) - and not ((v_to.has_ub() and var_val > v_to.ub)) - ): - v_to.set_value(var_val, skip_validation=True) - elif v_to.has_lb() and var_val < v_to.lb: - v_to.set_value(v_to.lb) - elif v_to.has_ub() and var_val > v_to.ub: - v_to.set_value(v_to.ub) - # Check to see if this is just a tolerance issue - elif ignore_integrality and v_to.is_integer(): - v_to.set_value(var_val, skip_validation=True) - elif v_to.is_integer() and ( - abs(var_val - rounded_val) <= config.integer_tolerance - ): - v_to.set_value(rounded_val, skip_validation=True) - elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: - v_to.set_value(0, skip_validation=True) - else: - raise ValueError("copy_var_list_values_from_solution_pool failed.") + copy_var_value(v_from, v_to, var_val, config, ignore_integrality) class GurobiPersistent4MindtPy(GurobiPersistent): @@ -983,6 +953,19 @@ def copy_var_list_values( """Copy variable values from one list to another. Rounds to Binary/Integer if necessary Sets to zero for NonNegativeReals if necessary + + from_list : list + The variables that provides the values to copy from. + to_list : list + The variables that need to set value. + config : ConfigBlock + The specific configurations for MindtPy. + skip_stale : bool, optional + Whether to skip the stale variables, by default False. + skip_fixed : bool, optional + Whether to skip the fixed variables, by default True. + ignore_integrality : bool, optional + Whether to ignore the integrality of integer variables, by default False. """ for v_from, v_to in zip(from_list, to_list): if skip_stale and v_from.stale: @@ -990,25 +973,59 @@ def copy_var_list_values( if skip_fixed and v_to.is_fixed(): continue # Skip fixed variables. var_val = value(v_from, exception=False) - rounded_val = int(round(var_val)) - if ( - var_val in v_to.domain - and not ((v_to.has_lb() and var_val < v_to.lb)) - and not ((v_to.has_ub() and var_val > v_to.ub)) - ): - v_to.set_value(value(v_from, exception=False)) - elif v_to.has_lb() and var_val < v_to.lb: - v_to.set_value(v_to.lb) - elif v_to.has_ub() and var_val > v_to.ub: - v_to.set_value(v_to.ub) - elif ignore_integrality and v_to.is_integer(): - v_to.set_value(value(v_from, exception=False), skip_validation=True) - elif v_to.is_integer() and ( - math.fabs(var_val - rounded_val) <= config.integer_tolerance + copy_var_value(v_from, v_to, var_val, config, ignore_integrality) + + +def copy_var_value(v_from, v_to, var_val, config, ignore_integrality): + """This function copies variable value from one to another. + Rounds to Binary/Integer if necessary. + Sets to zero for NonNegativeReals if necessary. + + NOTE: PEP 2180 changes the var behavior so that domain / + bounds violations no longer generate exceptions (and + instead log warnings). This means that the following will + always succeed and the ValueError should never be raised. + + Parameters + ---------- + v_from : Var + The variable that provides the values to copy from. + v_to : Var + The variable that needs to set value. + var_val : float + The value of v_to variable. + config : ConfigBlock + The specific configurations for MindtPy. + ignore_integrality : bool, optional + Whether to ignore the integrality of integer variables, by default False. + + Raises + ------ + ValueError + Cannot successfully set the value to variable v_to. + """ + # We don't want to trigger the reset of the global stale + # indicator, so we will set this variable to be "stale", + # knowing that set_value will switch it back to "not stale". + v_to.stale = True + rounded_val = int(round(var_val)) + if (var_val in v_to.domain + and not ((v_to.has_lb() and var_val < v_to.lb)) + and not ((v_to.has_ub() and var_val > v_to.ub)) ): - v_to.set_value(rounded_val) - elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: - v_to.set_value(0) - else: - raise ValueError("copy_var_list_values failed with variable {}, value = {} and rounded value = {}" - "".format(v_to.name, var_val, rounded_val)) + v_to.set_value(var_val) + elif v_to.has_lb() and var_val < v_to.lb: + v_to.set_value(v_to.lb) + elif v_to.has_ub() and var_val > v_to.ub: + v_to.set_value(v_to.ub) + elif ignore_integrality and v_to.is_integer(): + v_to.set_value(var_val, skip_validation=True) + elif v_to.is_integer() and ( + math.fabs(var_val - rounded_val) <= config.integer_tolerance + ): + v_to.set_value(rounded_val) + elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: + v_to.set_value(0) + else: + raise ValueError("copy_var_list_values failed with variable {}, value = {} and rounded value = {}" + "".format(v_to.name, var_val, rounded_val)) From dc41b8e969490e15d1c7948e386a8f2ba3ebedb8 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Mon, 27 Nov 2023 20:07:43 -0500 Subject: [PATCH 024/103] add exc_info for the error message --- pyomo/contrib/mindtpy/algorithm_base_class.py | 17 ++++++++++------- pyomo/contrib/mindtpy/cut_generation.py | 11 +++++++---- pyomo/contrib/mindtpy/extended_cutting_plane.py | 4 ++-- .../mindtpy/global_outer_approximation.py | 3 ++- pyomo/contrib/mindtpy/single_tree.py | 9 +++++---- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index 92e1075fe90..141e7f9f09f 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -802,7 +802,7 @@ def MindtPy_initialization(self): try: self.curr_int_sol = get_integer_solution(self.working_model) except TypeError as e: - config.logger.error(e) + config.logger.error(e, exc_info=True) raise ValueError( 'The initial integer combination is not provided or not complete. ' 'Please provide the complete integer combination or use other initialization strategy.' @@ -1083,7 +1083,7 @@ def solve_subproblem(self): 0, c_geq * (rhs - value(c.body)) ) except (ValueError, OverflowError) as e: - config.logger.error(e) + config.logger.error(e, exc_info=True) self.fixed_nlp.tmp_duals[c] = None evaluation_error = True if evaluation_error: @@ -1100,8 +1100,9 @@ def solve_subproblem(self): tolerance=config.constraint_tolerance, ) except InfeasibleConstraintException as e: + config.logger.error(e, exc_info=True) config.logger.error( - str(e) + '\nInfeasibility detected in deactivate_trivial_constraints.' + 'Infeasibility detected in deactivate_trivial_constraints.' ) results = SolverResults() results.solver.termination_condition = tc.infeasible @@ -1401,7 +1402,7 @@ def solve_feasibility_subproblem(self): if len(feas_soln.solution) > 0: feas_subproblem.solutions.load_from(feas_soln) except (ValueError, OverflowError) as e: - config.logger.error(e) + config.logger.error(e, exc_info=True) for nlp_var, orig_val in zip( MindtPy.variable_list, self.initial_var_values ): @@ -1542,8 +1543,9 @@ def fix_dual_bound(self, last_iter_cuts): try: self.dual_bound = self.stored_bound[self.primal_bound] except KeyError as e: + config.logger.error(e, exc_info=True) config.logger.error( - str(e) + '\nNo stored bound found. Bound fix failed.' + 'No stored bound found. Bound fix failed.' ) else: config.logger.info( @@ -1670,7 +1672,7 @@ def solve_main(self): if len(main_mip_results.solution) > 0: self.mip.solutions.load_from(main_mip_results) except (ValueError, AttributeError, RuntimeError) as e: - config.logger.error(e) + config.logger.error(e, exc_info=True) if config.single_tree: config.logger.warning('Single tree terminate.') if get_main_elapsed_time(self.timing) >= config.time_limit: @@ -2369,8 +2371,9 @@ def solve_fp_subproblem(self): tolerance=config.constraint_tolerance, ) except InfeasibleConstraintException as e: + config.logger.error(e, exc_info=True) config.logger.error( - str(e) + '\nInfeasibility detected in deactivate_trivial_constraints.' + 'Infeasibility detected in deactivate_trivial_constraints.' ) results = SolverResults() results.solver.termination_condition = tc.infeasible diff --git a/pyomo/contrib/mindtpy/cut_generation.py b/pyomo/contrib/mindtpy/cut_generation.py index 28d302104a3..343170aabac 100644 --- a/pyomo/contrib/mindtpy/cut_generation.py +++ b/pyomo/contrib/mindtpy/cut_generation.py @@ -271,8 +271,9 @@ def add_ecp_cuts( try: upper_slack = constr.uslack() except (ValueError, OverflowError) as e: + config.logger.error(e, exc_info=True) config.logger.error( - str(e) + '\nConstraint {} has caused either a ' + 'Constraint {} has caused either a ' 'ValueError or OverflowError.' '\n'.format(constr) ) @@ -300,8 +301,9 @@ def add_ecp_cuts( try: lower_slack = constr.lslack() except (ValueError, OverflowError) as e: + config.logger.error(e, exc_info=True) config.logger.error( - str(e) + '\nConstraint {} has caused either a ' + 'Constraint {} has caused either a ' 'ValueError or OverflowError.' '\n'.format(constr) ) @@ -424,9 +426,10 @@ def add_affine_cuts(target_model, config, timing): try: mc_eqn = mc(constr.body) except MCPP_Error as e: + config.logger.error(e, exc_info=True) config.logger.error( - '\nSkipping constraint %s due to MCPP error %s' - % (constr.name, str(e)) + 'Skipping constraint %s due to MCPP error' + % (constr.name) ) continue # skip to the next constraint diff --git a/pyomo/contrib/mindtpy/extended_cutting_plane.py b/pyomo/contrib/mindtpy/extended_cutting_plane.py index 446304b1361..3a09af155a0 100644 --- a/pyomo/contrib/mindtpy/extended_cutting_plane.py +++ b/pyomo/contrib/mindtpy/extended_cutting_plane.py @@ -140,7 +140,7 @@ def all_nonlinear_constraint_satisfied(self): lower_slack = nlc.lslack() except (ValueError, OverflowError) as e: # Set lower_slack (upper_slack below) less than -config.ecp_tolerance in this case. - config.logger.error(e) + config.logger.error(e, exc_info=True) lower_slack = -10 * config.ecp_tolerance if lower_slack < -config.ecp_tolerance: config.logger.debug( @@ -153,7 +153,7 @@ def all_nonlinear_constraint_satisfied(self): try: upper_slack = nlc.uslack() except (ValueError, OverflowError) as e: - config.logger.error(e) + config.logger.error(e, exc_info=True) upper_slack = -10 * config.ecp_tolerance if upper_slack < -config.ecp_tolerance: config.logger.debug( diff --git a/pyomo/contrib/mindtpy/global_outer_approximation.py b/pyomo/contrib/mindtpy/global_outer_approximation.py index dfb7ef54630..817fb0bf4a8 100644 --- a/pyomo/contrib/mindtpy/global_outer_approximation.py +++ b/pyomo/contrib/mindtpy/global_outer_approximation.py @@ -108,4 +108,5 @@ def deactivate_no_good_cuts_when_fixing_bound(self, no_good_cuts): if self.config.use_tabu_list: self.integer_list = self.integer_list[:valid_no_good_cuts_num] except KeyError as e: - self.config.logger.error(str(e) + '\nDeactivating no-good cuts failed.') + self.config.logger.error(e, exc_info=True) + self.config.logger.error('Deactivating no-good cuts failed.') diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index a5d4401d623..5485e0298f2 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -259,9 +259,10 @@ def add_lazy_affine_cuts(self, mindtpy_solver, config, opt): try: mc_eqn = mc(constr.body) except MCPP_Error as e: + config.logger.error(e, exc_info=True) config.logger.debug( - 'Skipping constraint %s due to MCPP error %s' - % (constr.name, str(e)) + 'Skipping constraint %s due to MCPP error' + % (constr.name) ) continue # skip to the next constraint # TODO: check if the value of ccSlope and cvSlope is not Nan or inf. If so, we skip this. @@ -696,9 +697,9 @@ def __call__(self): mindtpy_solver.mip, None, mindtpy_solver, config, opt ) except ValueError as e: + config.logger.error(e, exc_info=True) config.logger.error( - str(e) - + "\nUsually this error is caused by the MIP start solution causing a math domain error. " + "Usually this error is caused by the MIP start solution causing a math domain error. " "We will skip it." ) return From dbe9f490fd49e47ea5634c8425eeddd019d731ce Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Mon, 27 Nov 2023 20:34:04 -0500 Subject: [PATCH 025/103] black format --- pyomo/contrib/mindtpy/algorithm_base_class.py | 4 +--- pyomo/contrib/mindtpy/cut_generation.py | 3 +-- pyomo/contrib/mindtpy/single_tree.py | 9 ++++++--- pyomo/contrib/mindtpy/util.py | 11 +++++++---- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index 141e7f9f09f..b06a4c730b4 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -1544,9 +1544,7 @@ def fix_dual_bound(self, last_iter_cuts): self.dual_bound = self.stored_bound[self.primal_bound] except KeyError as e: config.logger.error(e, exc_info=True) - config.logger.error( - 'No stored bound found. Bound fix failed.' - ) + config.logger.error('No stored bound found. Bound fix failed.') else: config.logger.info( 'Solve the main problem without the last no_good cut to fix the bound.' diff --git a/pyomo/contrib/mindtpy/cut_generation.py b/pyomo/contrib/mindtpy/cut_generation.py index 343170aabac..e57cfd2eada 100644 --- a/pyomo/contrib/mindtpy/cut_generation.py +++ b/pyomo/contrib/mindtpy/cut_generation.py @@ -428,8 +428,7 @@ def add_affine_cuts(target_model, config, timing): except MCPP_Error as e: config.logger.error(e, exc_info=True) config.logger.error( - 'Skipping constraint %s due to MCPP error' - % (constr.name) + 'Skipping constraint %s due to MCPP error' % (constr.name) ) continue # skip to the next constraint diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 5485e0298f2..5e4e378d6c5 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -16,7 +16,11 @@ from pyomo.repn import generate_standard_repn import pyomo.core.expr as EXPR from math import copysign -from pyomo.contrib.mindtpy.util import get_integer_solution, copy_var_list_values, copy_var_value +from pyomo.contrib.mindtpy.util import ( + get_integer_solution, + copy_var_list_values, + copy_var_value, +) from pyomo.contrib.gdpopt.util import get_main_elapsed_time, time_code from pyomo.opt import TerminationCondition as tc from pyomo.core import minimize, value @@ -261,8 +265,7 @@ def add_lazy_affine_cuts(self, mindtpy_solver, config, opt): except MCPP_Error as e: config.logger.error(e, exc_info=True) config.logger.debug( - 'Skipping constraint %s due to MCPP error' - % (constr.name) + 'Skipping constraint %s due to MCPP error' % (constr.name) ) continue # skip to the next constraint # TODO: check if the value of ccSlope and cvSlope is not Nan or inf. If so, we skip this. diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 2970a805540..7e3fbe415d4 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -1009,10 +1009,11 @@ def copy_var_value(v_from, v_to, var_val, config, ignore_integrality): # knowing that set_value will switch it back to "not stale". v_to.stale = True rounded_val = int(round(var_val)) - if (var_val in v_to.domain + if ( + var_val in v_to.domain and not ((v_to.has_lb() and var_val < v_to.lb)) and not ((v_to.has_ub() and var_val > v_to.ub)) - ): + ): v_to.set_value(var_val) elif v_to.has_lb() and var_val < v_to.lb: v_to.set_value(v_to.lb) @@ -1027,5 +1028,7 @@ def copy_var_value(v_from, v_to, var_val, config, ignore_integrality): elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: v_to.set_value(0) else: - raise ValueError("copy_var_list_values failed with variable {}, value = {} and rounded value = {}" - "".format(v_to.name, var_val, rounded_val)) + raise ValueError( + "copy_var_list_values failed with variable {}, value = {} and rounded value = {}" + "".format(v_to.name, var_val, rounded_val) + ) From 4ac390e12fcfa3277a9808ff7f7325bfde808124 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Tue, 28 Nov 2023 09:34:22 -0500 Subject: [PATCH 026/103] change dir() to locals() --- pyomo/contrib/mindtpy/algorithm_base_class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index b06a4c730b4..d5d015d180d 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -1684,7 +1684,7 @@ def solve_main(self): 'No integer solution is found, so the CPLEX solver will report an error status. ' ) # Value error will be raised if the MIP problem is unbounded and appsi solver is used when loading solutions. Although the problem is unbounded, a valid result is provided and we do not return None to let the algorithm continue. - if 'main_mip_results' in dir(): + if 'main_mip_results' in locals(): return self.mip, main_mip_results else: return None, None From a755067a6276e62569308d5ce80ef47574eaf63b Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Tue, 28 Nov 2023 10:51:56 -0500 Subject: [PATCH 027/103] improve int_sol_2_cuts_ind --- pyomo/contrib/mindtpy/algorithm_base_class.py | 9 +++++---- pyomo/contrib/mindtpy/single_tree.py | 14 +++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index d5d015d180d..2eec150453f 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -108,7 +108,7 @@ def __init__(self, **kwds): self.curr_int_sol = [] self.should_terminate = False self.integer_list = [] - # dictionary {integer solution (list): cuts index (list)} + # dictionary {integer solution (list): [cuts begin index, cuts end index] (list)} self.int_sol_2_cuts_ind = dict() # Set up iteration counters @@ -810,9 +810,10 @@ def MindtPy_initialization(self): self.integer_list.append(self.curr_int_sol) fixed_nlp, fixed_nlp_result = self.solve_subproblem() self.handle_nlp_subproblem_tc(fixed_nlp, fixed_nlp_result) - self.int_sol_2_cuts_ind[self.curr_int_sol] = list( - range(1, len(self.mip.MindtPy_utils.cuts.oa_cuts) + 1) - ) + self.int_sol_2_cuts_ind[self.curr_int_sol] = [ + 1, + len(self.mip.MindtPy_utils.cuts.oa_cuts), + ] elif config.init_strategy == 'FP': self.init_rNLP() self.fp_loop() diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 5e4e378d6c5..4733843d6a2 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -900,9 +900,10 @@ def LazyOACallback_gurobi(cb_m, cb_opt, cb_where, mindtpy_solver, config): # Your callback should be prepared to cut off solutions that violate any of your lazy constraints, including those that have already been added. Node solutions will usually respect previously added lazy constraints, but not always. # https://www.gurobi.com/documentation/current/refman/cs_cb_addlazy.html # If this happens, MindtPy will look for the index of corresponding cuts, instead of solving the fixed-NLP again. - for ind in mindtpy_solver.int_sol_2_cuts_ind[ + begin_index, end_index = mindtpy_solver.int_sol_2_cuts_ind[ mindtpy_solver.curr_int_sol - ]: + ] + for ind in range(begin_index, end_index + 1): cb_opt.cbLazy(mindtpy_solver.mip.MindtPy_utils.cuts.oa_cuts[ind]) return else: @@ -917,11 +918,10 @@ def LazyOACallback_gurobi(cb_m, cb_opt, cb_where, mindtpy_solver, config): mindtpy_solver.handle_nlp_subproblem_tc(fixed_nlp, fixed_nlp_result, cb_opt) if config.strategy == 'OA': # store the cut index corresponding to current integer solution. - mindtpy_solver.int_sol_2_cuts_ind[mindtpy_solver.curr_int_sol] = list( - range( - cut_ind + 1, len(mindtpy_solver.mip.MindtPy_utils.cuts.oa_cuts) + 1 - ) - ) + mindtpy_solver.int_sol_2_cuts_ind[mindtpy_solver.curr_int_sol] = [ + cut_ind + 1, + len(mindtpy_solver.mip.MindtPy_utils.cuts.oa_cuts), + ] def handle_lazy_main_feasible_solution_gurobi(cb_m, cb_opt, mindtpy_solver, config): From d9d29bf04806a3d666cae8f6e20773440ed07928 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Tue, 28 Nov 2023 15:40:32 -0500 Subject: [PATCH 028/103] rename copy_var_value to set_var_value --- pyomo/contrib/mindtpy/single_tree.py | 10 +++++++-- pyomo/contrib/mindtpy/util.py | 32 ++++++++++++++++++---------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 4733843d6a2..481ff38df8f 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -19,7 +19,7 @@ from pyomo.contrib.mindtpy.util import ( get_integer_solution, copy_var_list_values, - copy_var_value, + set_var_value, ) from pyomo.contrib.gdpopt.util import get_main_elapsed_time, time_code from pyomo.opt import TerminationCondition as tc @@ -62,7 +62,13 @@ def copy_lazy_var_list_values( if skip_fixed and v_to.is_fixed(): continue # Skip fixed variables. v_val = self.get_values(opt._pyomo_var_to_solver_var_map[v_from]) - copy_var_value(v_from, v_to, v_val, config, ignore_integrality=False) + set_var_value( + v_to, + v_val, + config.integer_tolerance, + config.zero_tolerance, + ignore_integrality=False, + ) def add_lazy_oa_cuts( self, diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 7e3fbe415d4..ea22eb1ec3a 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -693,7 +693,13 @@ def copy_var_list_values_from_solution_pool( elif config.mip_solver == 'gurobi_persistent': solver_model.setParam(gurobipy.GRB.Param.SolutionNumber, solution_name) var_val = var_map[v_from].Xn - copy_var_value(v_from, v_to, var_val, config, ignore_integrality) + set_var_value( + v_to, + var_val, + config.integer_tolerance, + config.zero_tolerance, + ignore_integrality, + ) class GurobiPersistent4MindtPy(GurobiPersistent): @@ -973,10 +979,16 @@ def copy_var_list_values( if skip_fixed and v_to.is_fixed(): continue # Skip fixed variables. var_val = value(v_from, exception=False) - copy_var_value(v_from, v_to, var_val, config, ignore_integrality) + set_var_value( + v_to, + var_val, + config.integer_tolerance, + config.zero_tolerance, + ignore_integrality, + ) -def copy_var_value(v_from, v_to, var_val, config, ignore_integrality): +def set_var_value(v_to, var_val, integer_tolerance, zero_tolerance, ignore_integrality): """This function copies variable value from one to another. Rounds to Binary/Integer if necessary. Sets to zero for NonNegativeReals if necessary. @@ -988,14 +1000,14 @@ def copy_var_value(v_from, v_to, var_val, config, ignore_integrality): Parameters ---------- - v_from : Var - The variable that provides the values to copy from. v_to : Var The variable that needs to set value. var_val : float The value of v_to variable. - config : ConfigBlock - The specific configurations for MindtPy. + integer_tolerance: float + Tolerance on integral values. + zero_tolerance: float + Tolerance on variable equal to zero. ignore_integrality : bool, optional Whether to ignore the integrality of integer variables, by default False. @@ -1021,11 +1033,9 @@ def copy_var_value(v_from, v_to, var_val, config, ignore_integrality): v_to.set_value(v_to.ub) elif ignore_integrality and v_to.is_integer(): v_to.set_value(var_val, skip_validation=True) - elif v_to.is_integer() and ( - math.fabs(var_val - rounded_val) <= config.integer_tolerance - ): + elif v_to.is_integer() and (math.fabs(var_val - rounded_val) <= integer_tolerance): v_to.set_value(rounded_val) - elif abs(var_val) <= config.zero_tolerance and 0 in v_to.domain: + elif abs(var_val) <= zero_tolerance and 0 in v_to.domain: v_to.set_value(0) else: raise ValueError( From f04424e0d747d4d6986b9224e3ca8d7e4ad246ac Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Tue, 28 Nov 2023 15:57:37 -0500 Subject: [PATCH 029/103] add unit test for mindtpy --- pyomo/contrib/mindtpy/tests/unit_test.py | 70 ++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 pyomo/contrib/mindtpy/tests/unit_test.py diff --git a/pyomo/contrib/mindtpy/tests/unit_test.py b/pyomo/contrib/mindtpy/tests/unit_test.py new file mode 100644 index 00000000000..d9b2e494ab0 --- /dev/null +++ b/pyomo/contrib/mindtpy/tests/unit_test.py @@ -0,0 +1,70 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2022 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import pyomo.common.unittest as unittest +from pyomo.contrib.mindtpy.util import set_var_value + +from pyomo.environ import Var, Integers, ConcreteModel, Integers + + +class UnitTestMindtPy(unittest.TestCase): + def test_set_var_value(self): + m = ConcreteModel() + m.x1 = Var(within=Integers, bounds=(-1, 4), initialize=0) + + set_var_value( + m.x1, + var_val=5, + integer_tolerance=1e-6, + zero_tolerance=1e-6, + ignore_integrality=False, + ) + self.assertEqual(m.x1.value, 4) + + set_var_value( + m.x1, + var_val=-2, + integer_tolerance=1e-6, + zero_tolerance=1e-6, + ignore_integrality=False, + ) + self.assertEqual(m.x1.value, -1) + + set_var_value( + m.x1, + var_val=1.1, + integer_tolerance=1e-6, + zero_tolerance=1e-6, + ignore_integrality=True, + ) + self.assertEqual(m.x1.value, 1.1) + + set_var_value( + m.x1, + var_val=2.00000001, + integer_tolerance=1e-6, + zero_tolerance=1e-6, + ignore_integrality=False, + ) + self.assertEqual(m.x1.value, 2) + + set_var_value( + m.x1, + var_val=0.0000001, + integer_tolerance=1e-9, + zero_tolerance=1e-6, + ignore_integrality=False, + ) + self.assertEqual(m.x1.value, 0) + + +if __name__ == '__main__': + unittest.main() From ef6666085071f235458a36dbe3cb62192e391a2f Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 29 Nov 2023 17:28:51 -0500 Subject: [PATCH 030/103] improve var_val description --- pyomo/contrib/mindtpy/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index ea22eb1ec3a..51ed59e80a2 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -1003,7 +1003,7 @@ def set_var_value(v_to, var_val, integer_tolerance, zero_tolerance, ignore_integ v_to : Var The variable that needs to set value. var_val : float - The value of v_to variable. + The desired value to set for Var v_to. integer_tolerance: float Tolerance on integral values. zero_tolerance: float From 04ea15effcc83213c49a82b1230ffa3c0a945211 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 29 Nov 2023 17:31:55 -0500 Subject: [PATCH 031/103] rename set_var_value to set_var_valid_value --- pyomo/contrib/mindtpy/single_tree.py | 4 ++-- pyomo/contrib/mindtpy/tests/unit_test.py | 14 +++++++------- pyomo/contrib/mindtpy/util.py | 8 +++++--- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 481ff38df8f..c4d49e3afd6 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -19,7 +19,7 @@ from pyomo.contrib.mindtpy.util import ( get_integer_solution, copy_var_list_values, - set_var_value, + set_var_valid_value, ) from pyomo.contrib.gdpopt.util import get_main_elapsed_time, time_code from pyomo.opt import TerminationCondition as tc @@ -62,7 +62,7 @@ def copy_lazy_var_list_values( if skip_fixed and v_to.is_fixed(): continue # Skip fixed variables. v_val = self.get_values(opt._pyomo_var_to_solver_var_map[v_from]) - set_var_value( + set_var_valid_value( v_to, v_val, config.integer_tolerance, diff --git a/pyomo/contrib/mindtpy/tests/unit_test.py b/pyomo/contrib/mindtpy/tests/unit_test.py index d9b2e494ab0..baf5e16bb4b 100644 --- a/pyomo/contrib/mindtpy/tests/unit_test.py +++ b/pyomo/contrib/mindtpy/tests/unit_test.py @@ -10,17 +10,17 @@ # ___________________________________________________________________________ import pyomo.common.unittest as unittest -from pyomo.contrib.mindtpy.util import set_var_value +from pyomo.contrib.mindtpy.util import set_var_valid_value from pyomo.environ import Var, Integers, ConcreteModel, Integers class UnitTestMindtPy(unittest.TestCase): - def test_set_var_value(self): + def test_set_var_valid_value(self): m = ConcreteModel() m.x1 = Var(within=Integers, bounds=(-1, 4), initialize=0) - set_var_value( + set_var_valid_value( m.x1, var_val=5, integer_tolerance=1e-6, @@ -29,7 +29,7 @@ def test_set_var_value(self): ) self.assertEqual(m.x1.value, 4) - set_var_value( + set_var_valid_value( m.x1, var_val=-2, integer_tolerance=1e-6, @@ -38,7 +38,7 @@ def test_set_var_value(self): ) self.assertEqual(m.x1.value, -1) - set_var_value( + set_var_valid_value( m.x1, var_val=1.1, integer_tolerance=1e-6, @@ -47,7 +47,7 @@ def test_set_var_value(self): ) self.assertEqual(m.x1.value, 1.1) - set_var_value( + set_var_valid_value( m.x1, var_val=2.00000001, integer_tolerance=1e-6, @@ -56,7 +56,7 @@ def test_set_var_value(self): ) self.assertEqual(m.x1.value, 2) - set_var_value( + set_var_valid_value( m.x1, var_val=0.0000001, integer_tolerance=1e-9, diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 51ed59e80a2..f6cc0567286 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -693,7 +693,7 @@ def copy_var_list_values_from_solution_pool( elif config.mip_solver == 'gurobi_persistent': solver_model.setParam(gurobipy.GRB.Param.SolutionNumber, solution_name) var_val = var_map[v_from].Xn - set_var_value( + set_var_valid_value( v_to, var_val, config.integer_tolerance, @@ -979,7 +979,7 @@ def copy_var_list_values( if skip_fixed and v_to.is_fixed(): continue # Skip fixed variables. var_val = value(v_from, exception=False) - set_var_value( + set_var_valid_value( v_to, var_val, config.integer_tolerance, @@ -988,7 +988,9 @@ def copy_var_list_values( ) -def set_var_value(v_to, var_val, integer_tolerance, zero_tolerance, ignore_integrality): +def set_var_valid_value( + v_to, var_val, integer_tolerance, zero_tolerance, ignore_integrality +): """This function copies variable value from one to another. Rounds to Binary/Integer if necessary. Sets to zero for NonNegativeReals if necessary. From f84ff8d3429eb88bcd50021a8f4d22bcc691f2fb Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 29 Nov 2023 17:39:52 -0500 Subject: [PATCH 032/103] change v_to to var --- pyomo/contrib/mindtpy/util.py | 42 +++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index f6cc0567286..a9802a8bd1e 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -989,9 +989,9 @@ def copy_var_list_values( def set_var_valid_value( - v_to, var_val, integer_tolerance, zero_tolerance, ignore_integrality + var, var_val, integer_tolerance, zero_tolerance, ignore_integrality ): - """This function copies variable value from one to another. + """This function tries to set a valid value for variable with the given input. Rounds to Binary/Integer if necessary. Sets to zero for NonNegativeReals if necessary. @@ -1002,10 +1002,10 @@ def set_var_valid_value( Parameters ---------- - v_to : Var + var : Var The variable that needs to set value. var_val : float - The desired value to set for Var v_to. + The desired value to set for var. integer_tolerance: float Tolerance on integral values. zero_tolerance: float @@ -1016,31 +1016,31 @@ def set_var_valid_value( Raises ------ ValueError - Cannot successfully set the value to variable v_to. + Cannot successfully set the value to the variable. """ # We don't want to trigger the reset of the global stale # indicator, so we will set this variable to be "stale", # knowing that set_value will switch it back to "not stale". - v_to.stale = True + var.stale = True rounded_val = int(round(var_val)) if ( - var_val in v_to.domain - and not ((v_to.has_lb() and var_val < v_to.lb)) - and not ((v_to.has_ub() and var_val > v_to.ub)) + var_val in var.domain + and not ((var.has_lb() and var_val < var.lb)) + and not ((var.has_ub() and var_val > var.ub)) ): - v_to.set_value(var_val) - elif v_to.has_lb() and var_val < v_to.lb: - v_to.set_value(v_to.lb) - elif v_to.has_ub() and var_val > v_to.ub: - v_to.set_value(v_to.ub) - elif ignore_integrality and v_to.is_integer(): - v_to.set_value(var_val, skip_validation=True) - elif v_to.is_integer() and (math.fabs(var_val - rounded_val) <= integer_tolerance): - v_to.set_value(rounded_val) - elif abs(var_val) <= zero_tolerance and 0 in v_to.domain: - v_to.set_value(0) + var.set_value(var_val) + elif var.has_lb() and var_val < var.lb: + var.set_value(var.lb) + elif var.has_ub() and var_val > var.ub: + var.set_value(var.ub) + elif ignore_integrality and var.is_integer(): + var.set_value(var_val, skip_validation=True) + elif var.is_integer() and (math.fabs(var_val - rounded_val) <= integer_tolerance): + var.set_value(rounded_val) + elif abs(var_val) <= zero_tolerance and 0 in var.domain: + var.set_value(0) else: raise ValueError( "copy_var_list_values failed with variable {}, value = {} and rounded value = {}" - "".format(v_to.name, var_val, rounded_val) + "".format(var.name, var_val, rounded_val) ) From 83b28cb0d4216b337d8308d07ea090092a71a880 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 29 Nov 2023 17:42:10 -0500 Subject: [PATCH 033/103] move NOTE from docstring to comment --- pyomo/contrib/mindtpy/util.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index a9802a8bd1e..afcb129e40e 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -995,11 +995,6 @@ def set_var_valid_value( Rounds to Binary/Integer if necessary. Sets to zero for NonNegativeReals if necessary. - NOTE: PEP 2180 changes the var behavior so that domain / - bounds violations no longer generate exceptions (and - instead log warnings). This means that the following will - always succeed and the ValueError should never be raised. - Parameters ---------- var : Var @@ -1018,6 +1013,11 @@ def set_var_valid_value( ValueError Cannot successfully set the value to the variable. """ + # NOTE: PEP 2180 changes the var behavior so that domain + # bounds violations no longer generate exceptions (and + # instead log warnings). This means that the set_value method + # will always succeed and the ValueError should never be raised. + # We don't want to trigger the reset of the global stale # indicator, so we will set this variable to be "stale", # knowing that set_value will switch it back to "not stale". From 355df8b4a112597d4c1c45b0b6cd6a4ca13ff6af Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 30 Nov 2023 10:44:36 -0500 Subject: [PATCH 034/103] remove redundant test --- pyomo/repn/tests/ampl/test_nlv2.py | 34 ------------------------------ 1 file changed, 34 deletions(-) diff --git a/pyomo/repn/tests/ampl/test_nlv2.py b/pyomo/repn/tests/ampl/test_nlv2.py index 460c45b4ebb..fe5f422d323 100644 --- a/pyomo/repn/tests/ampl/test_nlv2.py +++ b/pyomo/repn/tests/ampl/test_nlv2.py @@ -1055,40 +1055,6 @@ def test_log_timing(self): re.sub(r'\d\.\d\d\]', '#.##]', LOG.getvalue()), ) - def test_log_timing(self): - # This tests an error possibly reported by #2810 - m = ConcreteModel() - m.x = Var(range(6)) - m.x[0].domain = pyo.Binary - m.x[1].domain = pyo.Integers - m.x[2].domain = pyo.Integers - m.p = Param(initialize=5, mutable=True) - m.o1 = Objective([1, 2], rule=lambda m, i: 1) - m.o2 = Objective(expr=m.x[1] * m.x[2]) - m.c1 = Constraint([1, 2], rule=lambda m, i: sum(m.x.values()) == 1) - m.c2 = Constraint(expr=m.p * m.x[1] ** 2 + m.x[2] ** 3 <= 100) - - self.maxDiff = None - OUT = io.StringIO() - with capture_output() as LOG: - with report_timing(level=logging.DEBUG): - nl_writer.NLWriter().write(m, OUT) - self.assertEqual( - """ [+ #.##] Initialized column order - [+ #.##] Collected suffixes - [+ #.##] Objective o1 - [+ #.##] Objective o2 - [+ #.##] Constraint c1 - [+ #.##] Constraint c2 - [+ #.##] Categorized model variables: 14 nnz - [+ #.##] Set row / column ordering: 6 var [3, 1, 2 R/B/Z], 3 con [2, 1 L/NL] - [+ #.##] Generated row/col labels & comments - [+ #.##] Wrote NL stream - [ #.##] Generated NL representation -""", - re.sub(r'\d\.\d\d\]', '#.##]', LOG.getvalue()), - ) - def test_linear_constraint_npv_const(self): # This tests an error possibly reported by #2810 m = ConcreteModel() From a7a01c229738fe680ad8d2f7f6814ab0e2c38a0c Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 30 Nov 2023 18:28:06 -0500 Subject: [PATCH 035/103] add test_add_var_bound --- pyomo/contrib/mindtpy/tests/unit_test.py | 31 ++++++++++++++++++++++++ pyomo/contrib/mindtpy/util.py | 4 +-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/mindtpy/tests/unit_test.py b/pyomo/contrib/mindtpy/tests/unit_test.py index baf5e16bb4b..a1ceadda41e 100644 --- a/pyomo/contrib/mindtpy/tests/unit_test.py +++ b/pyomo/contrib/mindtpy/tests/unit_test.py @@ -13,6 +13,10 @@ from pyomo.contrib.mindtpy.util import set_var_valid_value from pyomo.environ import Var, Integers, ConcreteModel, Integers +from pyomo.contrib.mindtpy.algorithm_base_class import _MindtPyAlgorithm +from pyomo.contrib.mindtpy.config_options import _get_MindtPy_OA_config +from pyomo.contrib.mindtpy.tests.MINLP5_simple import SimpleMINLP5 +from pyomo.contrib.mindtpy.util import add_var_bound class UnitTestMindtPy(unittest.TestCase): @@ -65,6 +69,33 @@ def test_set_var_valid_value(self): ) self.assertEqual(m.x1.value, 0) + def test_add_var_bound(self): + m = SimpleMINLP5().clone() + m.x.lb = None + m.x.ub = None + m.y.lb = None + m.y.ub = None + solver_object = _MindtPyAlgorithm() + solver_object.config = _get_MindtPy_OA_config() + solver_object.set_up_solve_data(m) + solver_object.create_utility_block(solver_object.working_model, 'MindtPy_utils') + add_var_bound(solver_object.working_model, solver_object.config) + self.assertEqual( + solver_object.working_model.x.lower, + -solver_object.config.continuous_var_bound - 1, + ) + self.assertEqual( + solver_object.working_model.x.upper, + solver_object.config.continuous_var_bound, + ) + self.assertEqual( + solver_object.working_model.y.lower, + -solver_object.config.integer_var_bound - 1, + ) + self.assertEqual( + solver_object.working_model.y.upper, solver_object.config.integer_var_bound + ) + if __name__ == '__main__': unittest.main() diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index afcb129e40e..1173dfe0cca 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -134,12 +134,12 @@ def add_var_bound(model, config): for var in EXPR.identify_variables(c.body): if var.has_lb() and var.has_ub(): continue - elif not var.has_lb(): + if not var.has_lb(): if var.is_integer(): var.setlb(-config.integer_var_bound - 1) else: var.setlb(-config.continuous_var_bound - 1) - elif not var.has_ub(): + if not var.has_ub(): if var.is_integer(): var.setub(config.integer_var_bound) else: From 875269fb7b7d5cdb3396ff1b7a2e5e2b5fc4e0d2 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 30 Nov 2023 18:29:28 -0500 Subject: [PATCH 036/103] delete redundant set_up_logger function --- pyomo/contrib/mindtpy/util.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 1173dfe0cca..ec2829c6a18 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -724,25 +724,6 @@ def f(gurobi_model, where): return f -def set_up_logger(config): - """Set up the formatter and handler for logger. - - Parameters - ---------- - config : ConfigBlock - The specific configurations for MindtPy. - """ - config.logger.handlers.clear() - config.logger.propagate = False - ch = logging.StreamHandler() - ch.setLevel(config.logging_level) - # create formatter and add it to the handlers - formatter = logging.Formatter('%(message)s') - ch.setFormatter(formatter) - # add the handlers to logger - config.logger.addHandler(ch) - - def epigraph_reformulation(exp, slack_var_list, constraint_list, use_mcpp, sense): """Epigraph reformulation. From 65e58f531c40fa466da60954b4bc5cc9b508055d Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 30 Nov 2023 19:50:14 -0500 Subject: [PATCH 037/103] add test_FP_L1_norm --- .../mindtpy/tests/test_mindtpy_feas_pump.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/tests/test_mindtpy_feas_pump.py b/pyomo/contrib/mindtpy/tests/test_mindtpy_feas_pump.py index 697a63d17c8..dcb5c4bce75 100644 --- a/pyomo/contrib/mindtpy/tests/test_mindtpy_feas_pump.py +++ b/pyomo/contrib/mindtpy/tests/test_mindtpy_feas_pump.py @@ -17,7 +17,7 @@ from pyomo.contrib.mindtpy.tests.feasibility_pump1 import FeasPump1 from pyomo.contrib.mindtpy.tests.feasibility_pump2 import FeasPump2 -required_solvers = ('ipopt', 'cplex') +required_solvers = ('ipopt', 'glpk') # TODO: 'appsi_highs' will fail here. if all(SolverFactory(s).available(exception_flag=False) for s in required_solvers): subsolvers_available = True @@ -69,6 +69,22 @@ def test_FP(self): log_infeasible_constraints(model) self.assertTrue(is_feasible(model, self.get_config(opt))) + def test_FP_L1_norm(self): + """Test the feasibility pump algorithm.""" + with SolverFactory('mindtpy') as opt: + for model in model_list: + model = model.clone() + results = opt.solve( + model, + strategy='FP', + mip_solver=required_solvers[1], + nlp_solver=required_solvers[0], + absolute_bound_tolerance=1e-5, + fp_main_norm='L1', + ) + log_infeasible_constraints(model) + self.assertTrue(is_feasible(model, self.get_config(opt))) + def test_FP_OA_8PP(self): """Test the FP-OA algorithm.""" with SolverFactory('mindtpy') as opt: From c8eead976a96ee87e79ce22bf8866e6b28abeb66 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 30 Nov 2023 20:08:16 -0500 Subject: [PATCH 038/103] improve mindtpy logging --- pyomo/contrib/mindtpy/algorithm_base_class.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index 2eec150453f..78250d1ba59 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -125,6 +125,9 @@ def __init__(self, **kwds): self.log_formatter = ( ' {:>9} {:>15} {:>15g} {:>12g} {:>12g} {:>7.2%} {:>7.2f}' ) + self.termination_condition_log_formatter = ( + ' {:>9} {:>15} {:>15} {:>12g} {:>12g} {:>7.2%} {:>7.2f}' + ) self.fixed_nlp_log_formatter = ( '{:1}{:>9} {:>15} {:>15g} {:>12g} {:>12g} {:>7.2%} {:>7.2f}' ) @@ -1919,11 +1922,6 @@ def handle_main_max_timelimit(self, main_mip, main_mip_results): """ # If we have found a valid feasible solution, we take that. If not, we can at least use the dual bound. MindtPy = main_mip.MindtPy_utils - self.config.logger.info( - 'Unable to optimize MILP main problem ' - 'within time limit. ' - 'Using current solver feasible solution.' - ) copy_var_list_values( main_mip.MindtPy_utils.variable_list, self.fixed_nlp.MindtPy_utils.variable_list, @@ -1932,10 +1930,10 @@ def handle_main_max_timelimit(self, main_mip, main_mip_results): ) self.update_suboptimal_dual_bound(main_mip_results) self.config.logger.info( - self.log_formatter.format( + self.termination_condition_log_formatter.format( self.mip_iter, 'MILP', - value(MindtPy.mip_obj.expr), + 'maxTimeLimit', self.primal_bound, self.dual_bound, self.rel_gap, @@ -1962,8 +1960,18 @@ def handle_main_unbounded(self, main_mip): # to the constraints, and deactivated for the linear main problem. config = self.config MindtPy = main_mip.MindtPy_utils + config.logger.info( + self.termination_condition_log_formatter.format( + self.mip_iter, + 'MILP', + 'Unbounded', + self.primal_bound, + self.dual_bound, + self.rel_gap, + get_main_elapsed_time(self.timing), + ) + ) config.logger.warning( - 'main MILP was unbounded. ' 'Resolving with arbitrary bound values of (-{0:.10g}, {0:.10g}) on the objective. ' 'You can change this bound with the option obj_bound.'.format( config.obj_bound From d22946164ff018f41d42d9f554090b46fedac371 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 7 Dec 2023 13:29:39 -0500 Subject: [PATCH 039/103] fix greybox cuts bug --- pyomo/contrib/mindtpy/cut_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/cut_generation.py b/pyomo/contrib/mindtpy/cut_generation.py index e57cfd2eada..4ee7a6ff07b 100644 --- a/pyomo/contrib/mindtpy/cut_generation.py +++ b/pyomo/contrib/mindtpy/cut_generation.py @@ -210,8 +210,8 @@ def add_oa_cuts_for_grey_box( target_model_grey_box.inputs.values() ) ) + - (output - value(output)) ) - - (output - value(output)) - (slack_var if config.add_slack else 0) <= 0 ) From 3d1db1363f5e96069ef5f6deafe458a83f0fb0fe Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 7 Dec 2023 16:21:59 -0500 Subject: [PATCH 040/103] redesign calc_jacobians function --- pyomo/contrib/mindtpy/extended_cutting_plane.py | 4 +++- pyomo/contrib/mindtpy/feasibility_pump.py | 4 +++- pyomo/contrib/mindtpy/outer_approximation.py | 4 +++- pyomo/contrib/mindtpy/util.py | 10 +++++----- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/mindtpy/extended_cutting_plane.py b/pyomo/contrib/mindtpy/extended_cutting_plane.py index 3a09af155a0..08c89a4c5f0 100644 --- a/pyomo/contrib/mindtpy/extended_cutting_plane.py +++ b/pyomo/contrib/mindtpy/extended_cutting_plane.py @@ -86,7 +86,9 @@ def check_config(self): def initialize_mip_problem(self): '''Deactivate the nonlinear constraints to create the MIP problem.''' super().initialize_mip_problem() - self.jacobians = calc_jacobians(self.mip, self.config) # preload jacobians + self.jacobians = calc_jacobians( + self.mip, self.config.differentiate_mode + ) # preload jacobians self.mip.MindtPy_utils.cuts.ecp_cuts = ConstraintList( doc='Extended Cutting Planes' ) diff --git a/pyomo/contrib/mindtpy/feasibility_pump.py b/pyomo/contrib/mindtpy/feasibility_pump.py index 990f56b8f93..bf6fb8f84bb 100644 --- a/pyomo/contrib/mindtpy/feasibility_pump.py +++ b/pyomo/contrib/mindtpy/feasibility_pump.py @@ -46,7 +46,9 @@ def check_config(self): def initialize_mip_problem(self): '''Deactivate the nonlinear constraints to create the MIP problem.''' super().initialize_mip_problem() - self.jacobians = calc_jacobians(self.mip, self.config) # preload jacobians + self.jacobians = calc_jacobians( + self.mip, self.config.differentiate_mode + ) # preload jacobians self.mip.MindtPy_utils.cuts.oa_cuts = ConstraintList( doc='Outer approximation cuts' ) diff --git a/pyomo/contrib/mindtpy/outer_approximation.py b/pyomo/contrib/mindtpy/outer_approximation.py index 6cf0b26cb37..4fd140a0bba 100644 --- a/pyomo/contrib/mindtpy/outer_approximation.py +++ b/pyomo/contrib/mindtpy/outer_approximation.py @@ -96,7 +96,9 @@ def check_config(self): def initialize_mip_problem(self): '''Deactivate the nonlinear constraints to create the MIP problem.''' super().initialize_mip_problem() - self.jacobians = calc_jacobians(self.mip, self.config) # preload jacobians + self.jacobians = calc_jacobians( + self.mip, self.config.differentiate_mode + ) # preload jacobians self.mip.MindtPy_utils.cuts.oa_cuts = ConstraintList( doc='Outer approximation cuts' ) diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index ec2829c6a18..5ca4604d37e 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -41,7 +41,7 @@ numpy = attempt_import('numpy')[0] -def calc_jacobians(model, config): +def calc_jacobians(model, differentiate_mode): """Generates a map of jacobians for the variables in the model. This function generates a map of jacobians corresponding to the variables in the @@ -51,15 +51,15 @@ def calc_jacobians(model, config): ---------- model : Pyomo model Target model to calculate jacobian. - config : ConfigBlock - The specific configurations for MindtPy. + differentiate_mode : String + The differentiate mode to calculate Jacobians. """ # Map nonlinear_constraint --> Map( # variable --> jacobian of constraint w.r.t. variable) jacobians = ComponentMap() - if config.differentiate_mode == 'reverse_symbolic': + if differentiate_mode == 'reverse_symbolic': mode = EXPR.differentiate.Modes.reverse_symbolic - elif config.differentiate_mode == 'sympy': + elif differentiate_mode == 'sympy': mode = EXPR.differentiate.Modes.sympy for c in model.MindtPy_utils.nonlinear_constraint_list: vars_in_constr = list(EXPR.identify_variables(c.body)) From 7e694136a8ac8cd197c7b56f2613a40597acbb59 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 7 Dec 2023 16:26:05 -0500 Subject: [PATCH 041/103] redesign initialize_feas_subproblem function --- pyomo/contrib/mindtpy/algorithm_base_class.py | 2 +- pyomo/contrib/mindtpy/util.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index 78250d1ba59..05f1e4389d3 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -2629,7 +2629,7 @@ def initialize_mip_problem(self): self.fixed_nlp = self.working_model.clone() TransformationFactory('core.fix_integer_vars').apply_to(self.fixed_nlp) - initialize_feas_subproblem(self.fixed_nlp, config) + initialize_feas_subproblem(self.fixed_nlp, config.feasibility_norm) def initialize_subsolvers(self): """Initialize and set options for MIP and NLP subsolvers.""" diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 5ca4604d37e..551945dfc67 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -70,7 +70,7 @@ def calc_jacobians(model, differentiate_mode): return jacobians -def initialize_feas_subproblem(m, config): +def initialize_feas_subproblem(m, feasibility_norm): """Adds feasibility slack variables according to config.feasibility_norm (given an infeasible problem). Defines the objective function of the feasibility subproblem. @@ -78,14 +78,14 @@ def initialize_feas_subproblem(m, config): ---------- m : Pyomo model The feasbility NLP subproblem. - config : ConfigBlock - The specific configurations for MindtPy. + feasibility_norm : String + The norm used to generate the objective function. """ MindtPy = m.MindtPy_utils # generate new constraints for i, constr in enumerate(MindtPy.nonlinear_constraint_list, 1): if constr.has_ub(): - if config.feasibility_norm in {'L1', 'L2'}: + if feasibility_norm in {'L1', 'L2'}: MindtPy.feas_opt.feas_constraints.add( constr.body - constr.upper <= MindtPy.feas_opt.slack_var[i] ) @@ -94,7 +94,7 @@ def initialize_feas_subproblem(m, config): constr.body - constr.upper <= MindtPy.feas_opt.slack_var ) if constr.has_lb(): - if config.feasibility_norm in {'L1', 'L2'}: + if feasibility_norm in {'L1', 'L2'}: MindtPy.feas_opt.feas_constraints.add( constr.body - constr.lower >= -MindtPy.feas_opt.slack_var[i] ) @@ -103,11 +103,11 @@ def initialize_feas_subproblem(m, config): constr.body - constr.lower >= -MindtPy.feas_opt.slack_var ) # Setup objective function for the feasibility subproblem. - if config.feasibility_norm == 'L1': + if feasibility_norm == 'L1': MindtPy.feas_obj = Objective( expr=sum(s for s in MindtPy.feas_opt.slack_var.values()), sense=minimize ) - elif config.feasibility_norm == 'L2': + elif feasibility_norm == 'L2': MindtPy.feas_obj = Objective( expr=sum(s * s for s in MindtPy.feas_opt.slack_var.values()), sense=minimize ) From c9f788849c0c15c197e85a8ce7e10df52bce2c98 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 7 Dec 2023 16:34:26 -0500 Subject: [PATCH 042/103] redesign calc_jacobians function --- pyomo/contrib/mindtpy/extended_cutting_plane.py | 3 ++- pyomo/contrib/mindtpy/feasibility_pump.py | 3 ++- pyomo/contrib/mindtpy/outer_approximation.py | 3 ++- pyomo/contrib/mindtpy/util.py | 10 +++++----- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/mindtpy/extended_cutting_plane.py b/pyomo/contrib/mindtpy/extended_cutting_plane.py index 08c89a4c5f0..f5fa205e091 100644 --- a/pyomo/contrib/mindtpy/extended_cutting_plane.py +++ b/pyomo/contrib/mindtpy/extended_cutting_plane.py @@ -87,7 +87,8 @@ def initialize_mip_problem(self): '''Deactivate the nonlinear constraints to create the MIP problem.''' super().initialize_mip_problem() self.jacobians = calc_jacobians( - self.mip, self.config.differentiate_mode + self.mip.MindtPy_utils.nonlinear_constraint_list, + self.config.differentiate_mode, ) # preload jacobians self.mip.MindtPy_utils.cuts.ecp_cuts = ConstraintList( doc='Extended Cutting Planes' diff --git a/pyomo/contrib/mindtpy/feasibility_pump.py b/pyomo/contrib/mindtpy/feasibility_pump.py index bf6fb8f84bb..9d5be89bab5 100644 --- a/pyomo/contrib/mindtpy/feasibility_pump.py +++ b/pyomo/contrib/mindtpy/feasibility_pump.py @@ -47,7 +47,8 @@ def initialize_mip_problem(self): '''Deactivate the nonlinear constraints to create the MIP problem.''' super().initialize_mip_problem() self.jacobians = calc_jacobians( - self.mip, self.config.differentiate_mode + self.mip.MindtPy_utils.nonlinear_constraint_list, + self.config.differentiate_mode, ) # preload jacobians self.mip.MindtPy_utils.cuts.oa_cuts = ConstraintList( doc='Outer approximation cuts' diff --git a/pyomo/contrib/mindtpy/outer_approximation.py b/pyomo/contrib/mindtpy/outer_approximation.py index 4fd140a0bba..6d790ce70d0 100644 --- a/pyomo/contrib/mindtpy/outer_approximation.py +++ b/pyomo/contrib/mindtpy/outer_approximation.py @@ -97,7 +97,8 @@ def initialize_mip_problem(self): '''Deactivate the nonlinear constraints to create the MIP problem.''' super().initialize_mip_problem() self.jacobians = calc_jacobians( - self.mip, self.config.differentiate_mode + self.mip.MindtPy_utils.nonlinear_constraint_list, + self.config.differentiate_mode, ) # preload jacobians self.mip.MindtPy_utils.cuts.oa_cuts = ConstraintList( doc='Outer approximation cuts' diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 551945dfc67..5845b3047f5 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -41,16 +41,16 @@ numpy = attempt_import('numpy')[0] -def calc_jacobians(model, differentiate_mode): +def calc_jacobians(constraint_list, differentiate_mode): """Generates a map of jacobians for the variables in the model. This function generates a map of jacobians corresponding to the variables in the - model. + constraint list. Parameters ---------- - model : Pyomo model - Target model to calculate jacobian. + constraint_list : List + The list of constraints to calculate Jacobians. differentiate_mode : String The differentiate mode to calculate Jacobians. """ @@ -61,7 +61,7 @@ def calc_jacobians(model, differentiate_mode): mode = EXPR.differentiate.Modes.reverse_symbolic elif differentiate_mode == 'sympy': mode = EXPR.differentiate.Modes.sympy - for c in model.MindtPy_utils.nonlinear_constraint_list: + for c in constraint_list: vars_in_constr = list(EXPR.identify_variables(c.body)) jac_list = EXPR.differentiate(c.body, wrt_list=vars_in_constr, mode=mode) jacobians[c] = ComponentMap( From c571f5c2d1952db77ea980d472a90d64463a355f Mon Sep 17 00:00:00 2001 From: robbybp Date: Tue, 12 Dec 2023 08:19:04 -0700 Subject: [PATCH 043/103] initial implementation of identify-via-amplrepn --- pyomo/contrib/incidence_analysis/incidence.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/pyomo/contrib/incidence_analysis/incidence.py b/pyomo/contrib/incidence_analysis/incidence.py index 1852cf75648..974153984d2 100644 --- a/pyomo/contrib/incidence_analysis/incidence.py +++ b/pyomo/contrib/incidence_analysis/incidence.py @@ -16,6 +16,8 @@ from pyomo.core.expr.visitor import identify_variables from pyomo.core.expr.numvalue import value as pyo_value from pyomo.repn import generate_standard_repn +from pyomo.repn.nl_writer import AMPLRepnVisitor, AMPLRepn, text_nl_template +from pyomo.repn.util import FileDeterminism, FileDeterminism_to_SortComponents from pyomo.util.subsystems import TemporarySubsystemManager from pyomo.contrib.incidence_analysis.config import IncidenceMethod, IncidenceConfig @@ -74,6 +76,45 @@ def _get_incident_via_standard_repn(expr, include_fixed, linear_only): return unique_variables +def _get_incident_via_amplrepn(expr, linear_only): + subexpression_cache = {} + subexpression_order = [] + external_functions = {} + var_map = {} + used_named_expressions = set() + symbolic_solver_labels = False + export_defined_variabels = True + sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) + visitor = AMPLRepnVisitor( + text_nl_template, + subexpression_cache, + subexpression_order, + external_functions, + var_map, + used_named_expressions, + symbolic_solver_labels, + export_defined_variables, + sorter, + ) + AMPLRepn.ActiveVisitor = visitor + try: + repn = visitor.walk_expression((expr, None, 0, 1.0)) + finally: + AMPLRepn.ActiveVisitor = None + + nonlinear_vars = [var_map[v_id] for v_id in repn.nonlinear[1]] + nonlinear_vid_set = set(repn.nonlinear[1]) + linear_only_vars = [ + var_map[v_id] for v_id, coef in repn.linear.items() + if coef != 0.0 and v_id not in nonlinear_vid_set + ] + if linear_only: + return linear_only_vars + else: + variables = linear_only_vars + nonlinear_vars + return variables + + def get_incident_variables(expr, **kwds): """Get variables that participate in an expression From 07c65e940a32a5fae3c6509c4a055c28ef4b146b Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Tue, 12 Dec 2023 09:53:14 -0700 Subject: [PATCH 044/103] add IncidenceMethod.ampl_repn option --- pyomo/contrib/incidence_analysis/config.py | 3 +++ pyomo/contrib/incidence_analysis/incidence.py | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/config.py b/pyomo/contrib/incidence_analysis/config.py index 56841617cac..60acc53abfc 100644 --- a/pyomo/contrib/incidence_analysis/config.py +++ b/pyomo/contrib/incidence_analysis/config.py @@ -24,6 +24,9 @@ class IncidenceMethod(enum.Enum): standard_repn = 1 """Use ``pyomo.repn.standard_repn.generate_standard_repn``""" + ampl_repn = 2 + """Use ``pyomo.repn.plugins.nl_writer.AMPLRepnVisitor``""" + _include_fixed = ConfigValue( default=False, diff --git a/pyomo/contrib/incidence_analysis/incidence.py b/pyomo/contrib/incidence_analysis/incidence.py index 974153984d2..f16b248463c 100644 --- a/pyomo/contrib/incidence_analysis/incidence.py +++ b/pyomo/contrib/incidence_analysis/incidence.py @@ -16,7 +16,7 @@ from pyomo.core.expr.visitor import identify_variables from pyomo.core.expr.numvalue import value as pyo_value from pyomo.repn import generate_standard_repn -from pyomo.repn.nl_writer import AMPLRepnVisitor, AMPLRepn, text_nl_template +from pyomo.repn.plugins.nl_writer import AMPLRepnVisitor, AMPLRepn, text_nl_template from pyomo.repn.util import FileDeterminism, FileDeterminism_to_SortComponents from pyomo.util.subsystems import TemporarySubsystemManager from pyomo.contrib.incidence_analysis.config import IncidenceMethod, IncidenceConfig @@ -76,14 +76,14 @@ def _get_incident_via_standard_repn(expr, include_fixed, linear_only): return unique_variables -def _get_incident_via_amplrepn(expr, linear_only): +def _get_incident_via_ampl_repn(expr, linear_only): subexpression_cache = {} subexpression_order = [] external_functions = {} var_map = {} used_named_expressions = set() symbolic_solver_labels = False - export_defined_variabels = True + export_defined_variables = True sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) visitor = AMPLRepnVisitor( text_nl_template, @@ -102,8 +102,9 @@ def _get_incident_via_amplrepn(expr, linear_only): finally: AMPLRepn.ActiveVisitor = None - nonlinear_vars = [var_map[v_id] for v_id in repn.nonlinear[1]] - nonlinear_vid_set = set(repn.nonlinear[1]) + nonlinear_var_ids = [] if repn.nonlinear is None else repn.nonlinear[1] + nonlinear_vars = [var_map[v_id] for v_id in nonlinear_var_ids] + nonlinear_vid_set = set(nonlinear_var_ids) linear_only_vars = [ var_map[v_id] for v_id, coef in repn.linear.items() if coef != 0.0 and v_id not in nonlinear_vid_set @@ -161,10 +162,16 @@ def get_incident_variables(expr, **kwds): raise RuntimeError( "linear_only=True is not supported when using identify_variables" ) + if include_fixed and method is IncidenceMethod.ampl_repn: + raise RuntimeError( + "include_fixed=True is not supported when using ampl_repn" + ) if method is IncidenceMethod.identify_variables: return _get_incident_via_identify_variables(expr, include_fixed) elif method is IncidenceMethod.standard_repn: return _get_incident_via_standard_repn(expr, include_fixed, linear_only) + elif method is IncidenceMethod.ampl_repn: + return _get_incident_via_ampl_repn(expr, linear_only) else: raise ValueError( f"Unrecognized value {method} for the method used to identify incident" From 5c88a9794ad2d05595eb849d53af3d18a50747ab Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Tue, 12 Dec 2023 09:53:57 -0700 Subject: [PATCH 045/103] refactor tests and test ampl_repn option --- .../tests/test_incidence.py | 124 ++++++++++++------ 1 file changed, 83 insertions(+), 41 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/tests/test_incidence.py b/pyomo/contrib/incidence_analysis/tests/test_incidence.py index 7f57dd904a7..e37e4f97691 100644 --- a/pyomo/contrib/incidence_analysis/tests/test_incidence.py +++ b/pyomo/contrib/incidence_analysis/tests/test_incidence.py @@ -63,37 +63,37 @@ def test_incidence_with_fixed_variable(self): var_set = ComponentSet(variables) self.assertEqual(var_set, ComponentSet([m.x[1], m.x[3]])) - def test_incidence_with_mutable_parameter(self): + +class _TestIncidenceLinearOnly(object): + def _get_incident_variables(self, expr): + raise NotImplementedError( + "_TestIncidenceLinearOnly should not be used directly" + ) + + def test_linear_only(self): m = pyo.ConcreteModel() m.x = pyo.Var([1, 2, 3]) - m.p = pyo.Param(mutable=True, initialize=None) - expr = m.x[1] + m.p * m.x[1] * m.x[2] + m.x[1] * pyo.exp(m.x[3]) - variables = self._get_incident_variables(expr) - self.assertEqual(ComponentSet(variables), ComponentSet(m.x[:])) + expr = 2 * m.x[1] + 4 * m.x[2] * m.x[1] - m.x[1] * pyo.exp(m.x[3]) + variables = self._get_incident_variables(expr, linear_only=True) + self.assertEqual(len(variables), 0) -class TestIncidenceStandardRepn(unittest.TestCase, _TestIncidence): - def _get_incident_variables(self, expr, **kwds): - method = IncidenceMethod.standard_repn - return get_incident_variables(expr, method=method, **kwds) + expr = 2 * m.x[1] + 2 * m.x[2] * m.x[3] + 3 * m.x[2] + variables = self._get_incident_variables(expr, linear_only=True) + self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1]])) - def test_assumed_standard_repn_behavior(self): - m = pyo.ConcreteModel() - m.x = pyo.Var([1, 2]) - m.p = pyo.Param(initialize=0.0) + m.x[3].fix(2.5) + expr = 2 * m.x[1] + 2 * m.x[2] * m.x[3] + 3 * m.x[2] + variables = self._get_incident_variables(expr, linear_only=True) + self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1], m.x[2]])) - # We rely on variables with constant coefficients of zero not appearing - # in the standard repn (as opposed to appearing with explicit - # coefficients of zero). - expr = m.x[1] + 0 * m.x[2] - repn = generate_standard_repn(expr) - self.assertEqual(len(repn.linear_vars), 1) - self.assertIs(repn.linear_vars[0], m.x[1]) - expr = m.p * m.x[1] + m.x[2] - repn = generate_standard_repn(expr) - self.assertEqual(len(repn.linear_vars), 1) - self.assertIs(repn.linear_vars[0], m.x[2]) +class _TestIncidenceLinearCancellation(object): + """Tests for methods that perform linear cancellation""" + def _get_incident_variables(self, expr): + raise NotImplementedError( + "_TestIncidenceLinearCancellation should not be used directly" + ) def test_zero_coef(self): m = pyo.ConcreteModel() @@ -113,23 +113,6 @@ def test_variable_minus_itself(self): var_set = ComponentSet(variables) self.assertEqual(var_set, ComponentSet([m.x[2], m.x[3]])) - def test_linear_only(self): - m = pyo.ConcreteModel() - m.x = pyo.Var([1, 2, 3]) - - expr = 2 * m.x[1] + 4 * m.x[2] * m.x[1] - m.x[1] * pyo.exp(m.x[3]) - variables = self._get_incident_variables(expr, linear_only=True) - self.assertEqual(len(variables), 0) - - expr = 2 * m.x[1] + 2 * m.x[2] * m.x[3] + 3 * m.x[2] - variables = self._get_incident_variables(expr, linear_only=True) - self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1]])) - - m.x[3].fix(2.5) - expr = 2 * m.x[1] + 2 * m.x[2] * m.x[3] + 3 * m.x[2] - variables = self._get_incident_variables(expr, linear_only=True) - self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1], m.x[2]])) - def test_fixed_zero_linear_coefficient(self): m = pyo.ConcreteModel() m.x = pyo.Var([1, 2, 3]) @@ -148,6 +131,9 @@ def test_fixed_zero_linear_coefficient(self): variables = self._get_incident_variables(expr) self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1], m.x[2]])) + # NOTE: This test assumes that all methods that support linear cancellation + # accept a linear_only argument. If this changes, this test wil need to be + # moved. def test_fixed_zero_coefficient_linear_only(self): m = pyo.ConcreteModel() m.x = pyo.Var([1, 2, 3]) @@ -159,6 +145,35 @@ def test_fixed_zero_coefficient_linear_only(self): self.assertEqual(len(variables), 1) self.assertIs(variables[0], m.x[3]) + +class TestIncidenceStandardRepn( + unittest.TestCase, + _TestIncidence, + _TestIncidenceLinearOnly, + _TestIncidenceLinearCancellation, +): + def _get_incident_variables(self, expr, **kwds): + method = IncidenceMethod.standard_repn + return get_incident_variables(expr, method=method, **kwds) + + def test_assumed_standard_repn_behavior(self): + m = pyo.ConcreteModel() + m.x = pyo.Var([1, 2]) + m.p = pyo.Param(initialize=0.0) + + # We rely on variables with constant coefficients of zero not appearing + # in the standard repn (as opposed to appearing with explicit + # coefficients of zero). + expr = m.x[1] + 0 * m.x[2] + repn = generate_standard_repn(expr) + self.assertEqual(len(repn.linear_vars), 1) + self.assertIs(repn.linear_vars[0], m.x[1]) + + expr = m.p * m.x[1] + m.x[2] + repn = generate_standard_repn(expr) + self.assertEqual(len(repn.linear_vars), 1) + self.assertIs(repn.linear_vars[0], m.x[2]) + def test_fixed_none_linear_coefficient(self): m = pyo.ConcreteModel() m.x = pyo.Var([1, 2, 3]) @@ -168,6 +183,14 @@ def test_fixed_none_linear_coefficient(self): variables = self._get_incident_variables(expr) self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1], m.x[2]])) + def test_incidence_with_mutable_parameter(self): + m = pyo.ConcreteModel() + m.x = pyo.Var([1, 2, 3]) + m.p = pyo.Param(mutable=True, initialize=None) + expr = m.x[1] + m.p * m.x[1] * m.x[2] + m.x[1] * pyo.exp(m.x[3]) + variables = self._get_incident_variables(expr) + self.assertEqual(ComponentSet(variables), ComponentSet(m.x[:])) + class TestIncidenceIdentifyVariables(unittest.TestCase, _TestIncidence): def _get_incident_variables(self, expr, **kwds): @@ -192,6 +215,25 @@ def test_variable_minus_itself(self): var_set = ComponentSet(variables) self.assertEqual(var_set, ComponentSet(m.x[:])) + def test_incidence_with_mutable_parameter(self): + m = pyo.ConcreteModel() + m.x = pyo.Var([1, 2, 3]) + m.p = pyo.Param(mutable=True, initialize=None) + expr = m.x[1] + m.p * m.x[1] * m.x[2] + m.x[1] * pyo.exp(m.x[3]) + variables = self._get_incident_variables(expr) + self.assertEqual(ComponentSet(variables), ComponentSet(m.x[:])) + + +class TestIncidenceAmplRepn( + unittest.TestCase, + _TestIncidence, + _TestIncidenceLinearOnly, + _TestIncidenceLinearCancellation, +): + def _get_incident_variables(self, expr, **kwds): + method = IncidenceMethod.ampl_repn + return get_incident_variables(expr, method=method, **kwds) + if __name__ == "__main__": unittest.main() From 515f7e59705ec6b95f6784f42a69ed2b1517161f Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Tue, 12 Dec 2023 12:09:44 -0700 Subject: [PATCH 046/103] apply black --- pyomo/contrib/incidence_analysis/incidence.py | 9 ++++----- pyomo/contrib/incidence_analysis/tests/test_incidence.py | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/incidence.py b/pyomo/contrib/incidence_analysis/incidence.py index f16b248463c..5ac7b49fa1f 100644 --- a/pyomo/contrib/incidence_analysis/incidence.py +++ b/pyomo/contrib/incidence_analysis/incidence.py @@ -103,10 +103,11 @@ def _get_incident_via_ampl_repn(expr, linear_only): AMPLRepn.ActiveVisitor = None nonlinear_var_ids = [] if repn.nonlinear is None else repn.nonlinear[1] - nonlinear_vars = [var_map[v_id] for v_id in nonlinear_var_ids] + nonlinear_vars = [var_map[v_id] for v_id in nonlinear_var_ids] nonlinear_vid_set = set(nonlinear_var_ids) linear_only_vars = [ - var_map[v_id] for v_id, coef in repn.linear.items() + var_map[v_id] + for v_id, coef in repn.linear.items() if coef != 0.0 and v_id not in nonlinear_vid_set ] if linear_only: @@ -163,9 +164,7 @@ def get_incident_variables(expr, **kwds): "linear_only=True is not supported when using identify_variables" ) if include_fixed and method is IncidenceMethod.ampl_repn: - raise RuntimeError( - "include_fixed=True is not supported when using ampl_repn" - ) + raise RuntimeError("include_fixed=True is not supported when using ampl_repn") if method is IncidenceMethod.identify_variables: return _get_incident_via_identify_variables(expr, include_fixed) elif method is IncidenceMethod.standard_repn: diff --git a/pyomo/contrib/incidence_analysis/tests/test_incidence.py b/pyomo/contrib/incidence_analysis/tests/test_incidence.py index e37e4f97691..bcf867c619a 100644 --- a/pyomo/contrib/incidence_analysis/tests/test_incidence.py +++ b/pyomo/contrib/incidence_analysis/tests/test_incidence.py @@ -90,6 +90,7 @@ def test_linear_only(self): class _TestIncidenceLinearCancellation(object): """Tests for methods that perform linear cancellation""" + def _get_incident_variables(self, expr): raise NotImplementedError( "_TestIncidenceLinearCancellation should not be used directly" From 8d5c737f551b3a513e13499957cc20dba86ec771 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Tue, 12 Dec 2023 12:10:58 -0700 Subject: [PATCH 047/103] add docstring to TestLinearOnly helper class --- pyomo/contrib/incidence_analysis/tests/test_incidence.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyomo/contrib/incidence_analysis/tests/test_incidence.py b/pyomo/contrib/incidence_analysis/tests/test_incidence.py index bcf867c619a..78493ecc651 100644 --- a/pyomo/contrib/incidence_analysis/tests/test_incidence.py +++ b/pyomo/contrib/incidence_analysis/tests/test_incidence.py @@ -65,6 +65,8 @@ def test_incidence_with_fixed_variable(self): class _TestIncidenceLinearOnly(object): + """Tests for methods that support linear_only""" + def _get_incident_variables(self, expr): raise NotImplementedError( "_TestIncidenceLinearOnly should not be used directly" From 45eb8616c67058b895ce49ebca9aa61877731976 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Tue, 12 Dec 2023 12:12:35 -0700 Subject: [PATCH 048/103] fix typo --- pyomo/contrib/incidence_analysis/tests/test_incidence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/incidence_analysis/tests/test_incidence.py b/pyomo/contrib/incidence_analysis/tests/test_incidence.py index 78493ecc651..87a9178dc1a 100644 --- a/pyomo/contrib/incidence_analysis/tests/test_incidence.py +++ b/pyomo/contrib/incidence_analysis/tests/test_incidence.py @@ -135,7 +135,7 @@ def test_fixed_zero_linear_coefficient(self): self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1], m.x[2]])) # NOTE: This test assumes that all methods that support linear cancellation - # accept a linear_only argument. If this changes, this test wil need to be + # accept a linear_only argument. If this changes, this test will need to be # moved. def test_fixed_zero_coefficient_linear_only(self): m = pyo.ConcreteModel() From 682e054a217cbc37a8c444efd38d2df491603cd3 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Tue, 12 Dec 2023 13:48:44 -0700 Subject: [PATCH 049/103] set export_defined_variables=False and add TODO comment about exploiting this later --- pyomo/contrib/incidence_analysis/incidence.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/incidence_analysis/incidence.py b/pyomo/contrib/incidence_analysis/incidence.py index 5ac7b49fa1f..b2cb23dc8c7 100644 --- a/pyomo/contrib/incidence_analysis/incidence.py +++ b/pyomo/contrib/incidence_analysis/incidence.py @@ -83,7 +83,10 @@ def _get_incident_via_ampl_repn(expr, linear_only): var_map = {} used_named_expressions = set() symbolic_solver_labels = False - export_defined_variables = True + # TODO: Explore potential performance benefit of exporting defined variables. + # This likely only shows up if we can preserve the subexpression cache across + # multiple constraint expressions. + export_defined_variables = False sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) visitor = AMPLRepnVisitor( text_nl_template, From c8ed1cda1a562788348157e72f90e103421af5d7 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Tue, 12 Dec 2023 13:53:09 -0700 Subject: [PATCH 050/103] add test that uses named expression --- pyomo/contrib/incidence_analysis/tests/test_incidence.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyomo/contrib/incidence_analysis/tests/test_incidence.py b/pyomo/contrib/incidence_analysis/tests/test_incidence.py index 87a9178dc1a..2354b0efc39 100644 --- a/pyomo/contrib/incidence_analysis/tests/test_incidence.py +++ b/pyomo/contrib/incidence_analysis/tests/test_incidence.py @@ -63,6 +63,15 @@ def test_incidence_with_fixed_variable(self): var_set = ComponentSet(variables) self.assertEqual(var_set, ComponentSet([m.x[1], m.x[3]])) + def test_incidence_with_named_expression(self): + m = pyo.ConcreteModel() + m.x = pyo.Var([1, 2, 3]) + m.subexpr = pyo.Expression(pyo.Integers) + m.subexpr[1] = m.x[1] * pyo.exp(m.x[3]) + expr = m.x[1] + m.x[1] * m.x[2] + m.subexpr[1] + variables = self._get_incident_variables(expr) + self.assertEqual(ComponentSet(variables), ComponentSet(m.x[:])) + class _TestIncidenceLinearOnly(object): """Tests for methods that support linear_only""" From 6675566fe7f4c7bf19832a5a72c09e7235cb6c8e Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Wed, 13 Dec 2023 13:19:16 -0700 Subject: [PATCH 051/103] re-use visitor when iterating over constraints --- pyomo/contrib/incidence_analysis/incidence.py | 61 ++++++++++--------- pyomo/contrib/incidence_analysis/interface.py | 39 ++++++++++-- 2 files changed, 67 insertions(+), 33 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/incidence.py b/pyomo/contrib/incidence_analysis/incidence.py index b2cb23dc8c7..7632f81e38a 100644 --- a/pyomo/contrib/incidence_analysis/incidence.py +++ b/pyomo/contrib/incidence_analysis/incidence.py @@ -76,34 +76,38 @@ def _get_incident_via_standard_repn(expr, include_fixed, linear_only): return unique_variables -def _get_incident_via_ampl_repn(expr, linear_only): - subexpression_cache = {} - subexpression_order = [] - external_functions = {} - var_map = {} - used_named_expressions = set() - symbolic_solver_labels = False - # TODO: Explore potential performance benefit of exporting defined variables. - # This likely only shows up if we can preserve the subexpression cache across - # multiple constraint expressions. - export_defined_variables = False - sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) - visitor = AMPLRepnVisitor( - text_nl_template, - subexpression_cache, - subexpression_order, - external_functions, - var_map, - used_named_expressions, - symbolic_solver_labels, - export_defined_variables, - sorter, - ) - AMPLRepn.ActiveVisitor = visitor - try: +def _get_incident_via_ampl_repn(expr, linear_only, visitor=None): + if visitor is None: + subexpression_cache = {} + subexpression_order = [] + external_functions = {} + var_map = {} + used_named_expressions = set() + symbolic_solver_labels = False + # TODO: Explore potential performance benefit of exporting defined variables. + # This likely only shows up if we can preserve the subexpression cache across + # multiple constraint expressions. + export_defined_variables = False + sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) + visitor = AMPLRepnVisitor( + text_nl_template, + subexpression_cache, + subexpression_order, + external_functions, + var_map, + used_named_expressions, + symbolic_solver_labels, + export_defined_variables, + sorter, + ) + AMPLRepn.ActiveVisitor = visitor + try: + repn = visitor.walk_expression((expr, None, 0, 1.0)) + finally: + AMPLRepn.ActiveVisitor = None + else: + var_map = visitor.var_map repn = visitor.walk_expression((expr, None, 0, 1.0)) - finally: - AMPLRepn.ActiveVisitor = None nonlinear_var_ids = [] if repn.nonlinear is None else repn.nonlinear[1] nonlinear_vars = [var_map[v_id] for v_id in nonlinear_var_ids] @@ -158,6 +162,7 @@ def get_incident_variables(expr, **kwds): ['x[1]', 'x[2]'] """ + visitor = kwds.pop("visitor", None) config = IncidenceConfig(kwds) method = config.method include_fixed = config.include_fixed @@ -173,7 +178,7 @@ def get_incident_variables(expr, **kwds): elif method is IncidenceMethod.standard_repn: return _get_incident_via_standard_repn(expr, include_fixed, linear_only) elif method is IncidenceMethod.ampl_repn: - return _get_incident_via_ampl_repn(expr, linear_only) + return _get_incident_via_ampl_repn(expr, linear_only, visitor=visitor) else: raise ValueError( f"Unrecognized value {method} for the method used to identify incident" diff --git a/pyomo/contrib/incidence_analysis/interface.py b/pyomo/contrib/incidence_analysis/interface.py index e922551c6a4..60e77d26f7a 100644 --- a/pyomo/contrib/incidence_analysis/interface.py +++ b/pyomo/contrib/incidence_analysis/interface.py @@ -29,7 +29,7 @@ plotly, ) from pyomo.common.deprecation import deprecated -from pyomo.contrib.incidence_analysis.config import IncidenceConfig +from pyomo.contrib.incidence_analysis.config import IncidenceConfig, IncidenceMethod from pyomo.contrib.incidence_analysis.matching import maximum_matching from pyomo.contrib.incidence_analysis.connected import get_independent_submatrices from pyomo.contrib.incidence_analysis.triangularize import ( @@ -45,6 +45,8 @@ ) from pyomo.contrib.incidence_analysis.incidence import get_incident_variables from pyomo.contrib.pynumero.asl import AmplInterface +from pyomo.repn.plugins.nl_writer import AMPLRepnVisitor, AMPLRepn, text_nl_template +from pyomo.repn.util import FileDeterminism, FileDeterminism_to_SortComponents pyomo_nlp, pyomo_nlp_available = attempt_import( 'pyomo.contrib.pynumero.interfaces.pyomo_nlp' @@ -99,10 +101,37 @@ def get_bipartite_incidence_graph(variables, constraints, **kwds): graph.add_nodes_from(range(M), bipartite=0) graph.add_nodes_from(range(M, M + N), bipartite=1) var_node_map = ComponentMap((v, M + i) for i, v in enumerate(variables)) - for i, con in enumerate(constraints): - for var in get_incident_variables(con.body, **config): - if var in var_node_map: - graph.add_edge(i, var_node_map[var]) + + if config.method == IncidenceMethod.ampl_repn: + subexpression_cache = {} + subexpression_order = [] + external_functions = {} + used_named_expressions = set() + symbolic_solver_labels = False + export_defined_variables = False + sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) + visitor = AMPLRepnVisitor( + text_nl_template, + subexpression_cache, + subexpression_order, + external_functions, + var_map, + used_named_expressions, + symbolic_solver_labels, + export_defined_variables, + sorter, + ) + else: + visitor = None + + AMPLRepn.ActiveVisitor = visitor + try: + for i, con in enumerate(constraints): + for var in get_incident_variables(con.body, visitor=visitor, **config): + if var in var_node_map: + graph.add_edge(i, var_node_map[var]) + finally: + AMPLRepn.ActiveVisitor = None return graph From bcd2435ebca336996893bfcb1243a54f3055d7f0 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Wed, 13 Dec 2023 13:40:40 -0700 Subject: [PATCH 052/103] add IncidenceMethod.standard_repn_compute_values option --- pyomo/contrib/incidence_analysis/config.py | 5 + pyomo/contrib/incidence_analysis/incidence.py | 16 ++- .../tests/test_incidence.py | 132 ++++++++++++------ 3 files changed, 111 insertions(+), 42 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/config.py b/pyomo/contrib/incidence_analysis/config.py index a107792a9cd..62856047121 100644 --- a/pyomo/contrib/incidence_analysis/config.py +++ b/pyomo/contrib/incidence_analysis/config.py @@ -24,6 +24,11 @@ class IncidenceMethod(enum.Enum): standard_repn = 1 """Use ``pyomo.repn.standard_repn.generate_standard_repn``""" + standard_repn_compute_values = 2 + """Use ``pyomo.repn.standard_repn.generate_standard_repn`` with + ``compute_values=True`` + """ + _include_fixed = ConfigValue( default=False, diff --git a/pyomo/contrib/incidence_analysis/incidence.py b/pyomo/contrib/incidence_analysis/incidence.py index 1852cf75648..b8dcd27c685 100644 --- a/pyomo/contrib/incidence_analysis/incidence.py +++ b/pyomo/contrib/incidence_analysis/incidence.py @@ -29,7 +29,9 @@ def _get_incident_via_identify_variables(expr, include_fixed): return list(identify_variables(expr, include_fixed=include_fixed)) -def _get_incident_via_standard_repn(expr, include_fixed, linear_only): +def _get_incident_via_standard_repn( + expr, include_fixed, linear_only, compute_values=False +): if include_fixed: to_unfix = [ var for var in identify_variables(expr, include_fixed=True) if var.fixed @@ -39,7 +41,9 @@ def _get_incident_via_standard_repn(expr, include_fixed, linear_only): context = nullcontext() with context: - repn = generate_standard_repn(expr, compute_values=False, quadratic=False) + repn = generate_standard_repn( + expr, compute_values=compute_values, quadratic=False + ) linear_vars = [] # Check coefficients to make sure we don't include linear variables with @@ -123,7 +127,13 @@ def get_incident_variables(expr, **kwds): if method is IncidenceMethod.identify_variables: return _get_incident_via_identify_variables(expr, include_fixed) elif method is IncidenceMethod.standard_repn: - return _get_incident_via_standard_repn(expr, include_fixed, linear_only) + return _get_incident_via_standard_repn( + expr, include_fixed, linear_only, compute_values=False + ) + elif method is IncidenceMethod.standard_repn_compute_values: + return _get_incident_via_standard_repn( + expr, include_fixed, linear_only, compute_values=True + ) else: raise ValueError( f"Unrecognized value {method} for the method used to identify incident" diff --git a/pyomo/contrib/incidence_analysis/tests/test_incidence.py b/pyomo/contrib/incidence_analysis/tests/test_incidence.py index 7f57dd904a7..b1a8ef1b14c 100644 --- a/pyomo/contrib/incidence_analysis/tests/test_incidence.py +++ b/pyomo/contrib/incidence_analysis/tests/test_incidence.py @@ -56,44 +56,56 @@ def test_basic_incidence(self): def test_incidence_with_fixed_variable(self): m = pyo.ConcreteModel() - m.x = pyo.Var([1, 2, 3]) + m.x = pyo.Var([1, 2, 3], initialize=1.0) expr = m.x[1] + m.x[1] * m.x[2] + m.x[1] * pyo.exp(m.x[3]) m.x[2].fix() variables = self._get_incident_variables(expr) var_set = ComponentSet(variables) self.assertEqual(var_set, ComponentSet([m.x[1], m.x[3]])) - def test_incidence_with_mutable_parameter(self): + def test_incidence_with_named_expression(self): m = pyo.ConcreteModel() m.x = pyo.Var([1, 2, 3]) - m.p = pyo.Param(mutable=True, initialize=None) - expr = m.x[1] + m.p * m.x[1] * m.x[2] + m.x[1] * pyo.exp(m.x[3]) + m.subexpr = pyo.Expression(pyo.Integers) + m.subexpr[1] = m.x[1] * pyo.exp(m.x[3]) + expr = m.x[1] + m.x[1] * m.x[2] + m.subexpr[1] variables = self._get_incident_variables(expr) self.assertEqual(ComponentSet(variables), ComponentSet(m.x[:])) -class TestIncidenceStandardRepn(unittest.TestCase, _TestIncidence): - def _get_incident_variables(self, expr, **kwds): - method = IncidenceMethod.standard_repn - return get_incident_variables(expr, method=method, **kwds) +class _TestIncidenceLinearOnly(object): + """Tests for methods that support linear_only""" - def test_assumed_standard_repn_behavior(self): + def _get_incident_variables(self, expr): + raise NotImplementedError( + "_TestIncidenceLinearOnly should not be used directly" + ) + + def test_linear_only(self): m = pyo.ConcreteModel() - m.x = pyo.Var([1, 2]) - m.p = pyo.Param(initialize=0.0) + m.x = pyo.Var([1, 2, 3]) - # We rely on variables with constant coefficients of zero not appearing - # in the standard repn (as opposed to appearing with explicit - # coefficients of zero). - expr = m.x[1] + 0 * m.x[2] - repn = generate_standard_repn(expr) - self.assertEqual(len(repn.linear_vars), 1) - self.assertIs(repn.linear_vars[0], m.x[1]) + expr = 2 * m.x[1] + 4 * m.x[2] * m.x[1] - m.x[1] * pyo.exp(m.x[3]) + variables = self._get_incident_variables(expr, linear_only=True) + self.assertEqual(len(variables), 0) - expr = m.p * m.x[1] + m.x[2] - repn = generate_standard_repn(expr) - self.assertEqual(len(repn.linear_vars), 1) - self.assertIs(repn.linear_vars[0], m.x[2]) + expr = 2 * m.x[1] + 2 * m.x[2] * m.x[3] + 3 * m.x[2] + variables = self._get_incident_variables(expr, linear_only=True) + self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1]])) + + m.x[3].fix(2.5) + expr = 2 * m.x[1] + 2 * m.x[2] * m.x[3] + 3 * m.x[2] + variables = self._get_incident_variables(expr, linear_only=True) + self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1], m.x[2]])) + + +class _TestIncidenceLinearCancellation(object): + """Tests for methods that perform linear cancellation""" + + def _get_incident_variables(self, expr): + raise NotImplementedError( + "_TestIncidenceLinearCancellation should not be used directly" + ) def test_zero_coef(self): m = pyo.ConcreteModel() @@ -113,23 +125,6 @@ def test_variable_minus_itself(self): var_set = ComponentSet(variables) self.assertEqual(var_set, ComponentSet([m.x[2], m.x[3]])) - def test_linear_only(self): - m = pyo.ConcreteModel() - m.x = pyo.Var([1, 2, 3]) - - expr = 2 * m.x[1] + 4 * m.x[2] * m.x[1] - m.x[1] * pyo.exp(m.x[3]) - variables = self._get_incident_variables(expr, linear_only=True) - self.assertEqual(len(variables), 0) - - expr = 2 * m.x[1] + 2 * m.x[2] * m.x[3] + 3 * m.x[2] - variables = self._get_incident_variables(expr, linear_only=True) - self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1]])) - - m.x[3].fix(2.5) - expr = 2 * m.x[1] + 2 * m.x[2] * m.x[3] + 3 * m.x[2] - variables = self._get_incident_variables(expr, linear_only=True) - self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1], m.x[2]])) - def test_fixed_zero_linear_coefficient(self): m = pyo.ConcreteModel() m.x = pyo.Var([1, 2, 3]) @@ -148,6 +143,9 @@ def test_fixed_zero_linear_coefficient(self): variables = self._get_incident_variables(expr) self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1], m.x[2]])) + # NOTE: This test assumes that all methods that support linear cancellation + # accept a linear_only argument. If this changes, this test will need to be + # moved. def test_fixed_zero_coefficient_linear_only(self): m = pyo.ConcreteModel() m.x = pyo.Var([1, 2, 3]) @@ -159,6 +157,35 @@ def test_fixed_zero_coefficient_linear_only(self): self.assertEqual(len(variables), 1) self.assertIs(variables[0], m.x[3]) + +class TestIncidenceStandardRepn( + unittest.TestCase, + _TestIncidence, + _TestIncidenceLinearOnly, + _TestIncidenceLinearCancellation, +): + def _get_incident_variables(self, expr, **kwds): + method = IncidenceMethod.standard_repn + return get_incident_variables(expr, method=method, **kwds) + + def test_assumed_standard_repn_behavior(self): + m = pyo.ConcreteModel() + m.x = pyo.Var([1, 2]) + m.p = pyo.Param(initialize=0.0) + + # We rely on variables with constant coefficients of zero not appearing + # in the standard repn (as opposed to appearing with explicit + # coefficients of zero). + expr = m.x[1] + 0 * m.x[2] + repn = generate_standard_repn(expr) + self.assertEqual(len(repn.linear_vars), 1) + self.assertIs(repn.linear_vars[0], m.x[1]) + + expr = m.p * m.x[1] + m.x[2] + repn = generate_standard_repn(expr) + self.assertEqual(len(repn.linear_vars), 1) + self.assertIs(repn.linear_vars[0], m.x[2]) + def test_fixed_none_linear_coefficient(self): m = pyo.ConcreteModel() m.x = pyo.Var([1, 2, 3]) @@ -168,6 +195,14 @@ def test_fixed_none_linear_coefficient(self): variables = self._get_incident_variables(expr) self.assertEqual(ComponentSet(variables), ComponentSet([m.x[1], m.x[2]])) + def test_incidence_with_mutable_parameter(self): + m = pyo.ConcreteModel() + m.x = pyo.Var([1, 2, 3]) + m.p = pyo.Param(mutable=True, initialize=None) + expr = m.x[1] + m.p * m.x[1] * m.x[2] + m.x[1] * pyo.exp(m.x[3]) + variables = self._get_incident_variables(expr) + self.assertEqual(ComponentSet(variables), ComponentSet(m.x[:])) + class TestIncidenceIdentifyVariables(unittest.TestCase, _TestIncidence): def _get_incident_variables(self, expr, **kwds): @@ -192,6 +227,25 @@ def test_variable_minus_itself(self): var_set = ComponentSet(variables) self.assertEqual(var_set, ComponentSet(m.x[:])) + def test_incidence_with_mutable_parameter(self): + m = pyo.ConcreteModel() + m.x = pyo.Var([1, 2, 3]) + m.p = pyo.Param(mutable=True, initialize=None) + expr = m.x[1] + m.p * m.x[1] * m.x[2] + m.x[1] * pyo.exp(m.x[3]) + variables = self._get_incident_variables(expr) + self.assertEqual(ComponentSet(variables), ComponentSet(m.x[:])) + + +class TestIncidenceStandardRepnComputeValues( + unittest.TestCase, + _TestIncidence, + _TestIncidenceLinearOnly, + _TestIncidenceLinearCancellation, +): + def _get_incident_variables(self, expr, **kwds): + method = IncidenceMethod.standard_repn_compute_values + return get_incident_variables(expr, method=method, **kwds) + if __name__ == "__main__": unittest.main() From ecaf0530ba1c85f75afd82a3662abe639d1bf8bf Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Wed, 13 Dec 2023 13:59:26 -0700 Subject: [PATCH 053/103] re-add var_map local variable --- pyomo/contrib/incidence_analysis/interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/contrib/incidence_analysis/interface.py b/pyomo/contrib/incidence_analysis/interface.py index 60e77d26f7a..726398f7750 100644 --- a/pyomo/contrib/incidence_analysis/interface.py +++ b/pyomo/contrib/incidence_analysis/interface.py @@ -106,6 +106,7 @@ def get_bipartite_incidence_graph(variables, constraints, **kwds): subexpression_cache = {} subexpression_order = [] external_functions = {} + var_map = {} used_named_expressions = set() symbolic_solver_labels = False export_defined_variables = False From 9236f4f1d1d0e8b49c7cbb3da37135ab0a2ad8e8 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Wed, 13 Dec 2023 13:59:50 -0700 Subject: [PATCH 054/103] filter duplicates from list of nonlinear vars --- pyomo/contrib/incidence_analysis/incidence.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/incidence.py b/pyomo/contrib/incidence_analysis/incidence.py index 7632f81e38a..feb8689a7c3 100644 --- a/pyomo/contrib/incidence_analysis/incidence.py +++ b/pyomo/contrib/incidence_analysis/incidence.py @@ -110,12 +110,18 @@ def _get_incident_via_ampl_repn(expr, linear_only, visitor=None): repn = visitor.walk_expression((expr, None, 0, 1.0)) nonlinear_var_ids = [] if repn.nonlinear is None else repn.nonlinear[1] - nonlinear_vars = [var_map[v_id] for v_id in nonlinear_var_ids] - nonlinear_vid_set = set(nonlinear_var_ids) + nonlinear_var_id_set = set() + unique_nonlinear_var_ids = [] + for v_id in nonlinear_var_ids: + if v_id not in nonlinear_var_id_set: + nonlinear_var_id_set.add(v_id) + unique_nonlinear_var_ids.append(v_id) + + nonlinear_vars = [var_map[v_id] for v_id in unique_nonlinear_var_ids] linear_only_vars = [ var_map[v_id] for v_id, coef in repn.linear.items() - if coef != 0.0 and v_id not in nonlinear_vid_set + if coef != 0.0 and v_id not in nonlinear_var_id_set ] if linear_only: return linear_only_vars From c04264005f6c7729b7a2054e8c7a96c389f13ef8 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Wed, 13 Dec 2023 14:31:33 -0700 Subject: [PATCH 055/103] re-use visitor in _generate_variables_in_constraints --- pyomo/contrib/incidence_analysis/interface.py | 45 ++++++++++++++++--- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/interface.py b/pyomo/contrib/incidence_analysis/interface.py index 726398f7750..ce5f4780210 100644 --- a/pyomo/contrib/incidence_analysis/interface.py +++ b/pyomo/contrib/incidence_analysis/interface.py @@ -194,12 +194,45 @@ def extract_bipartite_subgraph(graph, nodes0, nodes1): def _generate_variables_in_constraints(constraints, **kwds): config = IncidenceConfig(kwds) - known_vars = ComponentSet() - for con in constraints: - for var in get_incident_variables(con.body, **config): - if var not in known_vars: - known_vars.add(var) - yield var + + if config.method == IncidenceMethod.ampl_repn: + subexpression_cache = {} + subexpression_order = [] + external_functions = {} + var_map = {} + used_named_expressions = set() + symbolic_solver_labels = False + export_defined_variables = False + sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) + visitor = AMPLRepnVisitor( + text_nl_template, + subexpression_cache, + subexpression_order, + external_functions, + var_map, + used_named_expressions, + symbolic_solver_labels, + export_defined_variables, + sorter, + ) + else: + visitor = None + + AMPLRepn.ActiveVisitor = visitor + try: + known_vars = ComponentSet() + for con in constraints: + for var in get_incident_variables(con.body, visitor=visitor, **config): + if var not in known_vars: + known_vars.add(var) + yield var + finally: + # NOTE: I believe this is only guaranteed to be called when the + # generator is garbage collected. This could lead to some nasty + # bug where ActiveVisitor is set for longer than we intend. + # TODO: Convert this into a function. (or yield from variables + # after this try/finally. + AMPLRepn.ActiveVisitor = None def get_structural_incidence_matrix(variables, constraints, **kwds): From 76fee13fae1bd0888dbadee083aa13d3bb437786 Mon Sep 17 00:00:00 2001 From: robbybp Date: Sat, 6 Jan 2024 15:52:52 -0700 Subject: [PATCH 056/103] move AMPLRepnVisitor construction into ConfigValue validation --- pyomo/contrib/incidence_analysis/config.py | 95 ++++++++++++++++++- pyomo/contrib/incidence_analysis/incidence.py | 42 ++------ pyomo/contrib/incidence_analysis/interface.py | 85 +++-------------- 3 files changed, 116 insertions(+), 106 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/config.py b/pyomo/contrib/incidence_analysis/config.py index db9accbddc4..31b2bd3fc22 100644 --- a/pyomo/contrib/incidence_analysis/config.py +++ b/pyomo/contrib/incidence_analysis/config.py @@ -13,6 +13,9 @@ import enum from pyomo.common.config import ConfigDict, ConfigValue, InEnum +from pyomo.common.modeling import NOTSET +from pyomo.repn.plugins.nl_writer import AMPLRepnVisitor, AMPLRepn, text_nl_template +from pyomo.repn.util import FileDeterminism, FileDeterminism_to_SortComponents class IncidenceMethod(enum.Enum): @@ -62,7 +65,92 @@ class IncidenceMethod(enum.Enum): ) -IncidenceConfig = ConfigDict() +class _ReconstructVisitor: + pass + + +def _amplrepnvisitor_validator(visitor=_ReconstructVisitor): + # This checks for and returns a valid AMPLRepnVisitor, but I don't want + # to construct this if we're not using IncidenceMethod.ampl_repn. + # It is not necessarily the end of the world if we construct this, however, + # as the code should still work. + if visitor is _ReconstructVisitor: + subexpression_cache = {} + subexpression_order = [] + external_functions = {} + var_map = {} + used_named_expressions = set() + symbolic_solver_labels = False + # TODO: Explore potential performance benefit of exporting defined variables. + # This likely only shows up if we can preserve the subexpression cache across + # multiple constraint expressions. + export_defined_variables = False + sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) + amplvisitor = AMPLRepnVisitor( + text_nl_template, + subexpression_cache, + subexpression_order, + external_functions, + var_map, + used_named_expressions, + symbolic_solver_labels, + export_defined_variables, + sorter, + ) + elif not isinstance(visitor, AMPLRepnVisitor): + raise TypeError( + "'visitor' config argument should be an instance of AMPLRepnVisitor" + ) + else: + amplvisitor = visitor + return amplvisitor + + +_ampl_repn_visitor = ConfigValue( + default=_ReconstructVisitor, + domain=_amplrepnvisitor_validator, + description="Visitor used to generate AMPLRepn of each constraint", +) + + +class _IncidenceConfigDict(ConfigDict): + + def __call__( + self, + value=NOTSET, + default=NOTSET, + domain=NOTSET, + description=NOTSET, + doc=NOTSET, + visibility=NOTSET, + implicit=NOTSET, + implicit_domain=NOTSET, + preserve_implicit=False, + ): + init_value = value + new = super().__call__( + value=value, + default=default, + domain=domain, + description=description, + doc=doc, + visibility=visibility, + implicit=implicit, + implicit_domain=implicit_domain, + preserve_implicit=preserve_implicit, + ) + + if ( + new.method == IncidenceMethod.ampl_repn + and "ampl_repn_visitor" not in init_value + ): + new.ampl_repn_visitor = _ReconstructVisitor + + return new + + + +IncidenceConfig = _IncidenceConfigDict() """Options for incidence graph generation - ``include_fixed`` -- Flag indicating whether fixed variables should be included @@ -71,6 +159,8 @@ class IncidenceMethod(enum.Enum): should be included. - ``method`` -- Method used to identify incident variables. Must be a value of the ``IncidenceMethod`` enum. +- ``ampl_repn_visitor`` -- Expression visitor used to generate ``AMPLRepn`` of each + constraint. Must be an instance of ``AMPLRepnVisitor``. """ @@ -82,3 +172,6 @@ class IncidenceMethod(enum.Enum): IncidenceConfig.declare("method", _method) + + +IncidenceConfig.declare("ampl_repn_visitor", _ampl_repn_visitor) diff --git a/pyomo/contrib/incidence_analysis/incidence.py b/pyomo/contrib/incidence_analysis/incidence.py index 62ba7a0aec7..17307e89600 100644 --- a/pyomo/contrib/incidence_analysis/incidence.py +++ b/pyomo/contrib/incidence_analysis/incidence.py @@ -80,38 +80,14 @@ def _get_incident_via_standard_repn( return unique_variables -def _get_incident_via_ampl_repn(expr, linear_only, visitor=None): - if visitor is None: - subexpression_cache = {} - subexpression_order = [] - external_functions = {} - var_map = {} - used_named_expressions = set() - symbolic_solver_labels = False - # TODO: Explore potential performance benefit of exporting defined variables. - # This likely only shows up if we can preserve the subexpression cache across - # multiple constraint expressions. - export_defined_variables = False - sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) - visitor = AMPLRepnVisitor( - text_nl_template, - subexpression_cache, - subexpression_order, - external_functions, - var_map, - used_named_expressions, - symbolic_solver_labels, - export_defined_variables, - sorter, - ) - AMPLRepn.ActiveVisitor = visitor - try: - repn = visitor.walk_expression((expr, None, 0, 1.0)) - finally: - AMPLRepn.ActiveVisitor = None - else: - var_map = visitor.var_map +def _get_incident_via_ampl_repn(expr, linear_only, visitor): + var_map = visitor.var_map + orig_activevisitor = AMPLRepn.ActiveVisitor + AMPLRepn.ActiveVisitor = visitor + try: repn = visitor.walk_expression((expr, None, 0, 1.0)) + finally: + AMPLRepn.ActiveVisitor = orig_activevisitor nonlinear_var_ids = [] if repn.nonlinear is None else repn.nonlinear[1] nonlinear_var_id_set = set() @@ -172,11 +148,11 @@ def get_incident_variables(expr, **kwds): ['x[1]', 'x[2]'] """ - visitor = kwds.pop("visitor", None) config = IncidenceConfig(kwds) method = config.method include_fixed = config.include_fixed linear_only = config.linear_only + amplrepnvisitor = config.ampl_repn_visitor if linear_only and method is IncidenceMethod.identify_variables: raise RuntimeError( "linear_only=True is not supported when using identify_variables" @@ -194,7 +170,7 @@ def get_incident_variables(expr, **kwds): expr, include_fixed, linear_only, compute_values=True ) elif method is IncidenceMethod.ampl_repn: - return _get_incident_via_ampl_repn(expr, linear_only, visitor=visitor) + return _get_incident_via_ampl_repn(expr, linear_only, amplrepnvisitor) else: raise ValueError( f"Unrecognized value {method} for the method used to identify incident" diff --git a/pyomo/contrib/incidence_analysis/interface.py b/pyomo/contrib/incidence_analysis/interface.py index ce5f4780210..b8a6c1275f9 100644 --- a/pyomo/contrib/incidence_analysis/interface.py +++ b/pyomo/contrib/incidence_analysis/interface.py @@ -93,6 +93,8 @@ def get_bipartite_incidence_graph(variables, constraints, **kwds): ``networkx.Graph`` """ + # Note that this ConfigDict contains the visitor that we will re-use + # when constructing constraints. config = IncidenceConfig(kwds) _check_unindexed(variables + constraints) N = len(variables) @@ -101,38 +103,10 @@ def get_bipartite_incidence_graph(variables, constraints, **kwds): graph.add_nodes_from(range(M), bipartite=0) graph.add_nodes_from(range(M, M + N), bipartite=1) var_node_map = ComponentMap((v, M + i) for i, v in enumerate(variables)) - - if config.method == IncidenceMethod.ampl_repn: - subexpression_cache = {} - subexpression_order = [] - external_functions = {} - var_map = {} - used_named_expressions = set() - symbolic_solver_labels = False - export_defined_variables = False - sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) - visitor = AMPLRepnVisitor( - text_nl_template, - subexpression_cache, - subexpression_order, - external_functions, - var_map, - used_named_expressions, - symbolic_solver_labels, - export_defined_variables, - sorter, - ) - else: - visitor = None - - AMPLRepn.ActiveVisitor = visitor - try: - for i, con in enumerate(constraints): - for var in get_incident_variables(con.body, visitor=visitor, **config): - if var in var_node_map: - graph.add_edge(i, var_node_map[var]) - finally: - AMPLRepn.ActiveVisitor = None + for i, con in enumerate(constraints): + for var in get_incident_variables(con.body, **config): + if var in var_node_map: + graph.add_edge(i, var_node_map[var]) return graph @@ -193,46 +167,14 @@ def extract_bipartite_subgraph(graph, nodes0, nodes1): def _generate_variables_in_constraints(constraints, **kwds): + # Note: We construct a visitor here config = IncidenceConfig(kwds) - - if config.method == IncidenceMethod.ampl_repn: - subexpression_cache = {} - subexpression_order = [] - external_functions = {} - var_map = {} - used_named_expressions = set() - symbolic_solver_labels = False - export_defined_variables = False - sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) - visitor = AMPLRepnVisitor( - text_nl_template, - subexpression_cache, - subexpression_order, - external_functions, - var_map, - used_named_expressions, - symbolic_solver_labels, - export_defined_variables, - sorter, - ) - else: - visitor = None - - AMPLRepn.ActiveVisitor = visitor - try: - known_vars = ComponentSet() - for con in constraints: - for var in get_incident_variables(con.body, visitor=visitor, **config): - if var not in known_vars: - known_vars.add(var) - yield var - finally: - # NOTE: I believe this is only guaranteed to be called when the - # generator is garbage collected. This could lead to some nasty - # bug where ActiveVisitor is set for longer than we intend. - # TODO: Convert this into a function. (or yield from variables - # after this try/finally. - AMPLRepn.ActiveVisitor = None + known_vars = ComponentSet() + for con in constraints: + for var in get_incident_variables(con.body, **config): + if var not in known_vars: + known_vars.add(var) + yield var def get_structural_incidence_matrix(variables, constraints, **kwds): @@ -329,7 +271,6 @@ class IncidenceGraphInterface(object): ``evaluate_jacobian_eq`` method instead of ``evaluate_jacobian`` rather than checking constraint expression types. - """ def __init__(self, model=None, active=True, include_inequality=True, **kwds): From e3ddb015059458899c4e1cc492dbe88d08e8a7ad Mon Sep 17 00:00:00 2001 From: robbybp Date: Sat, 6 Jan 2024 15:54:57 -0700 Subject: [PATCH 057/103] remove whitespace --- pyomo/contrib/incidence_analysis/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/config.py b/pyomo/contrib/incidence_analysis/config.py index 31b2bd3fc22..036c563ae75 100644 --- a/pyomo/contrib/incidence_analysis/config.py +++ b/pyomo/contrib/incidence_analysis/config.py @@ -114,7 +114,6 @@ def _amplrepnvisitor_validator(visitor=_ReconstructVisitor): class _IncidenceConfigDict(ConfigDict): - def __call__( self, value=NOTSET, @@ -149,7 +148,6 @@ def __call__( return new - IncidenceConfig = _IncidenceConfigDict() """Options for incidence graph generation From a93d793e49f9f209f188d5a7ee1a72829a7423c4 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 10 Jan 2024 14:33:02 -0500 Subject: [PATCH 058/103] change all ''' to """ --- pyomo/contrib/mindtpy/algorithm_base_class.py | 6 +++--- pyomo/contrib/mindtpy/extended_cutting_plane.py | 2 +- pyomo/contrib/mindtpy/feasibility_pump.py | 2 +- pyomo/contrib/mindtpy/global_outer_approximation.py | 2 +- pyomo/contrib/mindtpy/outer_approximation.py | 2 +- pyomo/contrib/mindtpy/tests/MINLP_simple_grey_box.py | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index 05f1e4389d3..d732e95c422 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -519,9 +519,9 @@ def get_primal_integral(self): return primal_integral def get_integral_info(self): - ''' + """ Obtain primal integral, dual integral and primal dual gap integral. - ''' + """ self.primal_integral = self.get_primal_integral() self.dual_integral = self.get_dual_integral() self.primal_dual_gap_integral = self.primal_integral + self.dual_integral @@ -2598,7 +2598,7 @@ def fp_loop(self): self.working_model.MindtPy_utils.cuts.del_component('fp_orthogonality_cuts') def initialize_mip_problem(self): - '''Deactivate the nonlinear constraints to create the MIP problem.''' + """Deactivate the nonlinear constraints to create the MIP problem.""" # if single tree is activated, we need to add bounds for unbounded variables in nonlinear constraints to avoid unbounded main problem. config = self.config if config.single_tree: diff --git a/pyomo/contrib/mindtpy/extended_cutting_plane.py b/pyomo/contrib/mindtpy/extended_cutting_plane.py index f5fa205e091..ac13e352e35 100644 --- a/pyomo/contrib/mindtpy/extended_cutting_plane.py +++ b/pyomo/contrib/mindtpy/extended_cutting_plane.py @@ -84,7 +84,7 @@ def check_config(self): super().check_config() def initialize_mip_problem(self): - '''Deactivate the nonlinear constraints to create the MIP problem.''' + """Deactivate the nonlinear constraints to create the MIP problem.""" super().initialize_mip_problem() self.jacobians = calc_jacobians( self.mip.MindtPy_utils.nonlinear_constraint_list, diff --git a/pyomo/contrib/mindtpy/feasibility_pump.py b/pyomo/contrib/mindtpy/feasibility_pump.py index 9d5be89bab5..a34cceb014c 100644 --- a/pyomo/contrib/mindtpy/feasibility_pump.py +++ b/pyomo/contrib/mindtpy/feasibility_pump.py @@ -44,7 +44,7 @@ def check_config(self): super().check_config() def initialize_mip_problem(self): - '''Deactivate the nonlinear constraints to create the MIP problem.''' + """Deactivate the nonlinear constraints to create the MIP problem.""" super().initialize_mip_problem() self.jacobians = calc_jacobians( self.mip.MindtPy_utils.nonlinear_constraint_list, diff --git a/pyomo/contrib/mindtpy/global_outer_approximation.py b/pyomo/contrib/mindtpy/global_outer_approximation.py index 817fb0bf4a8..70fc4cffb90 100644 --- a/pyomo/contrib/mindtpy/global_outer_approximation.py +++ b/pyomo/contrib/mindtpy/global_outer_approximation.py @@ -67,7 +67,7 @@ def check_config(self): super().check_config() def initialize_mip_problem(self): - '''Deactivate the nonlinear constraints to create the MIP problem.''' + """Deactivate the nonlinear constraints to create the MIP problem.""" super().initialize_mip_problem() self.mip.MindtPy_utils.cuts.aff_cuts = ConstraintList(doc='Affine cuts') diff --git a/pyomo/contrib/mindtpy/outer_approximation.py b/pyomo/contrib/mindtpy/outer_approximation.py index 6d790ce70d0..f6e6147724e 100644 --- a/pyomo/contrib/mindtpy/outer_approximation.py +++ b/pyomo/contrib/mindtpy/outer_approximation.py @@ -94,7 +94,7 @@ def check_config(self): _MindtPyAlgorithm.check_config(self) def initialize_mip_problem(self): - '''Deactivate the nonlinear constraints to create the MIP problem.''' + """Deactivate the nonlinear constraints to create the MIP problem.""" super().initialize_mip_problem() self.jacobians = calc_jacobians( self.mip.MindtPy_utils.nonlinear_constraint_list, diff --git a/pyomo/contrib/mindtpy/tests/MINLP_simple_grey_box.py b/pyomo/contrib/mindtpy/tests/MINLP_simple_grey_box.py index 547efc0a74c..9c1f33e80cc 100644 --- a/pyomo/contrib/mindtpy/tests/MINLP_simple_grey_box.py +++ b/pyomo/contrib/mindtpy/tests/MINLP_simple_grey_box.py @@ -114,7 +114,7 @@ def evaluate_jacobian_equality_constraints(self): """Evaluate the Jacobian of the equality constraints.""" return None - ''' + """ def _extract_and_assemble_fim(self): M = np.zeros((self.n_parameters, self.n_parameters)) for i in range(self.n_parameters): @@ -122,7 +122,7 @@ def _extract_and_assemble_fim(self): M[i,k] = self._input_values[self.ele_to_order[(i,k)]] return M - ''' + """ def evaluate_jacobian_outputs(self): """Evaluate the Jacobian of the outputs.""" From 666cbaa4e776947032e86887473f1296a9baacff Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 10 Jan 2024 14:39:32 -0500 Subject: [PATCH 059/103] rename int_sol_2_cuts_ind to integer_solution_to_cuts_index --- pyomo/contrib/mindtpy/algorithm_base_class.py | 8 ++++---- pyomo/contrib/mindtpy/single_tree.py | 9 ++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index d732e95c422..7e8d390976c 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -102,14 +102,14 @@ def __init__(self, **kwds): self.fixed_nlp = None # We store bounds, timing info, iteration count, incumbent, and the - # expression of the original (possibly nonlinear) objective function. + # Expression of the original (possibly nonlinear) objective function. self.results = SolverResults() self.timing = Bunch() self.curr_int_sol = [] self.should_terminate = False self.integer_list = [] - # dictionary {integer solution (list): [cuts begin index, cuts end index] (list)} - self.int_sol_2_cuts_ind = dict() + # Dictionary {integer solution (list): [cuts begin index, cuts end index] (list)} + self.integer_solution_to_cuts_index = dict() # Set up iteration counters self.nlp_iter = 0 @@ -813,7 +813,7 @@ def MindtPy_initialization(self): self.integer_list.append(self.curr_int_sol) fixed_nlp, fixed_nlp_result = self.solve_subproblem() self.handle_nlp_subproblem_tc(fixed_nlp, fixed_nlp_result) - self.int_sol_2_cuts_ind[self.curr_int_sol] = [ + self.integer_solution_to_cuts_index[self.curr_int_sol] = [ 1, len(self.mip.MindtPy_utils.cuts.oa_cuts), ] diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index c4d49e3afd6..10b1f21ec5c 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -906,7 +906,7 @@ def LazyOACallback_gurobi(cb_m, cb_opt, cb_where, mindtpy_solver, config): # Your callback should be prepared to cut off solutions that violate any of your lazy constraints, including those that have already been added. Node solutions will usually respect previously added lazy constraints, but not always. # https://www.gurobi.com/documentation/current/refman/cs_cb_addlazy.html # If this happens, MindtPy will look for the index of corresponding cuts, instead of solving the fixed-NLP again. - begin_index, end_index = mindtpy_solver.int_sol_2_cuts_ind[ + begin_index, end_index = mindtpy_solver.integer_solution_to_cuts_index[ mindtpy_solver.curr_int_sol ] for ind in range(begin_index, end_index + 1): @@ -924,10 +924,9 @@ def LazyOACallback_gurobi(cb_m, cb_opt, cb_where, mindtpy_solver, config): mindtpy_solver.handle_nlp_subproblem_tc(fixed_nlp, fixed_nlp_result, cb_opt) if config.strategy == 'OA': # store the cut index corresponding to current integer solution. - mindtpy_solver.int_sol_2_cuts_ind[mindtpy_solver.curr_int_sol] = [ - cut_ind + 1, - len(mindtpy_solver.mip.MindtPy_utils.cuts.oa_cuts), - ] + mindtpy_solver.integer_solution_to_cuts_index[ + mindtpy_solver.curr_int_sol + ] = [cut_ind + 1, len(mindtpy_solver.mip.MindtPy_utils.cuts.oa_cuts)] def handle_lazy_main_feasible_solution_gurobi(cb_m, cb_opt, mindtpy_solver, config): From 14ebdcdc8a03c947164b64729a24e53c4b1b02db Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 10 Jan 2024 15:46:05 -0500 Subject: [PATCH 060/103] add one more comment to CPLEX lazy constraint callback --- pyomo/contrib/mindtpy/single_tree.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 10b1f21ec5c..145d85e0d37 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -666,6 +666,7 @@ def __call__(self): main_mip = self.main_mip mindtpy_solver = self.mindtpy_solver + # The lazy constraint callback may be invoked during MIP start processing. In that case get_solution_source returns mip_start_solution. # Reference: https://www.ibm.com/docs/en/icos/22.1.1?topic=SSSA5P_22.1.1/ilog.odms.cplex.help/refpythoncplex/html/cplex.callbacks.SolutionSource-class.htm # Another solution source is user_solution = 118, but it will not be encountered in LazyConstraintCallback. config.logger.info( From b143e87de6eb464cfec2b190114c6f068c9433ae Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 10 Jan 2024 15:46:51 -0500 Subject: [PATCH 061/103] remove the finished TODO --- pyomo/contrib/mindtpy/single_tree.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 145d85e0d37..09b5e704f75 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -274,7 +274,6 @@ def add_lazy_affine_cuts(self, mindtpy_solver, config, opt): 'Skipping constraint %s due to MCPP error' % (constr.name) ) continue # skip to the next constraint - # TODO: check if the value of ccSlope and cvSlope is not Nan or inf. If so, we skip this. ccSlope = mc_eqn.subcc() cvSlope = mc_eqn.subcv() ccStart = mc_eqn.concave() From 09bda47d460033736c2789e0a8e6d5dbaf581d6a Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 10 Jan 2024 15:48:44 -0500 Subject: [PATCH 062/103] add TODO for self.abort() --- pyomo/contrib/mindtpy/single_tree.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 09b5e704f75..228810a8f90 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -689,6 +689,7 @@ def __call__(self): mindtpy_solver.mip_start_lazy_oa_cuts = [] if mindtpy_solver.should_terminate: + # TODO: check the performance difference if we don't use self.abort() and let cplex terminate by itself. self.abort() return self.handle_lazy_main_feasible_solution(main_mip, mindtpy_solver, config, opt) @@ -744,6 +745,7 @@ def __call__(self): ) ) mindtpy_solver.results.solver.termination_condition = tc.optimal + # TODO: check the performance difference if we don't use self.abort() and let cplex terminate by itself. self.abort() return From 317dae89cdb4eac00e006bc808d32d887a14d787 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 11 Jan 2024 14:30:08 -0500 Subject: [PATCH 063/103] update the version of MindtPy --- pyomo/contrib/mindtpy/MindtPy.py | 8 ++++++++ pyomo/contrib/mindtpy/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/MindtPy.py b/pyomo/contrib/mindtpy/MindtPy.py index 6eb27c4c649..bd873d950fd 100644 --- a/pyomo/contrib/mindtpy/MindtPy.py +++ b/pyomo/contrib/mindtpy/MindtPy.py @@ -50,6 +50,14 @@ - Add single-tree implementation. - Add support for cplex_persistent solver. - Fix bug in OA cut expression in cut_generation.py. + +24.1.11 changes: +- fix gurobi single tree termination check bug +- fix Gurobi single tree cycle handling +- fix bug in feasibility pump method +- add special handling for infeasible relaxed NLP +- update the log format of infeasible fixed NLP subproblems +- create a new copy_var_list_values function """ from pyomo.contrib.mindtpy import __version__ diff --git a/pyomo/contrib/mindtpy/__init__.py b/pyomo/contrib/mindtpy/__init__.py index 8e2c2d9eaa4..8dcd085211f 100644 --- a/pyomo/contrib/mindtpy/__init__.py +++ b/pyomo/contrib/mindtpy/__init__.py @@ -1 +1 @@ -__version__ = (0, 1, 0) +__version__ = (1, 0, 0) From 7d64e1725b1af5173c9ff94cb885913ee1c834c2 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Mon, 22 Jan 2024 15:10:07 -0700 Subject: [PATCH 064/103] add get_config_from_kwds to use instead of hacking ConfigDict --- pyomo/contrib/incidence_analysis/config.py | 90 +++++++++++-------- pyomo/contrib/incidence_analysis/incidence.py | 18 ++-- pyomo/contrib/incidence_analysis/interface.py | 12 +-- 3 files changed, 70 insertions(+), 50 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/config.py b/pyomo/contrib/incidence_analysis/config.py index 036c563ae75..4ab086da508 100644 --- a/pyomo/contrib/incidence_analysis/config.py +++ b/pyomo/contrib/incidence_analysis/config.py @@ -69,45 +69,16 @@ class _ReconstructVisitor: pass -def _amplrepnvisitor_validator(visitor=_ReconstructVisitor): - # This checks for and returns a valid AMPLRepnVisitor, but I don't want - # to construct this if we're not using IncidenceMethod.ampl_repn. - # It is not necessarily the end of the world if we construct this, however, - # as the code should still work. - if visitor is _ReconstructVisitor: - subexpression_cache = {} - subexpression_order = [] - external_functions = {} - var_map = {} - used_named_expressions = set() - symbolic_solver_labels = False - # TODO: Explore potential performance benefit of exporting defined variables. - # This likely only shows up if we can preserve the subexpression cache across - # multiple constraint expressions. - export_defined_variables = False - sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) - amplvisitor = AMPLRepnVisitor( - text_nl_template, - subexpression_cache, - subexpression_order, - external_functions, - var_map, - used_named_expressions, - symbolic_solver_labels, - export_defined_variables, - sorter, - ) - elif not isinstance(visitor, AMPLRepnVisitor): +def _amplrepnvisitor_validator(visitor): + if not isinstance(visitor, AMPLRepnVisitor): raise TypeError( "'visitor' config argument should be an instance of AMPLRepnVisitor" ) - else: - amplvisitor = visitor - return amplvisitor + return visitor _ampl_repn_visitor = ConfigValue( - default=_ReconstructVisitor, + default=None, domain=_amplrepnvisitor_validator, description="Visitor used to generate AMPLRepn of each constraint", ) @@ -141,14 +112,14 @@ def __call__( if ( new.method == IncidenceMethod.ampl_repn - and "ampl_repn_visitor" not in init_value + and "_ampl_repn_visitor" not in init_value ): - new.ampl_repn_visitor = _ReconstructVisitor + new._ampl_repn_visitor = _ReconstructVisitor return new -IncidenceConfig = _IncidenceConfigDict() +IncidenceConfig = ConfigDict() """Options for incidence graph generation - ``include_fixed`` -- Flag indicating whether fixed variables should be included @@ -157,8 +128,9 @@ def __call__( should be included. - ``method`` -- Method used to identify incident variables. Must be a value of the ``IncidenceMethod`` enum. -- ``ampl_repn_visitor`` -- Expression visitor used to generate ``AMPLRepn`` of each - constraint. Must be an instance of ``AMPLRepnVisitor``. +- ``_ampl_repn_visitor`` -- Expression visitor used to generate ``AMPLRepn`` of each + constraint. Must be an instance of ``AMPLRepnVisitor``. *This option is constructed + automatically when needed and should not be set by users!* """ @@ -172,4 +144,44 @@ def __call__( IncidenceConfig.declare("method", _method) -IncidenceConfig.declare("ampl_repn_visitor", _ampl_repn_visitor) +IncidenceConfig.declare("_ampl_repn_visitor", _ampl_repn_visitor) + + +def get_config_from_kwds(**kwds): + """Get an instance of IncidenceConfig from provided keyword arguments. + + If the ``method`` argument is ``IncidenceMethod.ampl_repn`` and no + ``AMPLRepnVisitor`` has been provided, a new ``AMPLRepnVisitor`` is + constructed. This function should generally be used by callers such + as ``IncidenceGraphInterface`` to ensure that a visitor is created then + re-used when calling ``get_incident_variables`` in a loop. + + """ + if ( + kwds.get("method", None) is IncidenceMethod.ampl_repn + and kwds.get("_ampl_repn_visitor", None) is None + ): + subexpression_cache = {} + subexpression_order = [] + external_functions = {} + var_map = {} + used_named_expressions = set() + symbolic_solver_labels = False + # TODO: Explore potential performance benefit of exporting defined variables. + # This likely only shows up if we can preserve the subexpression cache across + # multiple constraint expressions. + export_defined_variables = False + sorter = FileDeterminism_to_SortComponents(FileDeterminism.ORDERED) + amplvisitor = AMPLRepnVisitor( + text_nl_template, + subexpression_cache, + subexpression_order, + external_functions, + var_map, + used_named_expressions, + symbolic_solver_labels, + export_defined_variables, + sorter, + ) + kwds["_ampl_repn_visitor"] = amplvisitor + return IncidenceConfig(kwds) diff --git a/pyomo/contrib/incidence_analysis/incidence.py b/pyomo/contrib/incidence_analysis/incidence.py index 17307e89600..1fc3380fe6b 100644 --- a/pyomo/contrib/incidence_analysis/incidence.py +++ b/pyomo/contrib/incidence_analysis/incidence.py @@ -19,7 +19,9 @@ from pyomo.repn.plugins.nl_writer import AMPLRepnVisitor, AMPLRepn, text_nl_template from pyomo.repn.util import FileDeterminism, FileDeterminism_to_SortComponents from pyomo.util.subsystems import TemporarySubsystemManager -from pyomo.contrib.incidence_analysis.config import IncidenceMethod, IncidenceConfig +from pyomo.contrib.incidence_analysis.config import ( + IncidenceMethod, get_config_from_kwds +) # @@ -148,17 +150,24 @@ def get_incident_variables(expr, **kwds): ['x[1]', 'x[2]'] """ - config = IncidenceConfig(kwds) + config = get_config_from_kwds(**kwds) method = config.method include_fixed = config.include_fixed linear_only = config.linear_only - amplrepnvisitor = config.ampl_repn_visitor + amplrepnvisitor = config._ampl_repn_visitor + + # Check compatibility of arguments if linear_only and method is IncidenceMethod.identify_variables: raise RuntimeError( "linear_only=True is not supported when using identify_variables" ) if include_fixed and method is IncidenceMethod.ampl_repn: raise RuntimeError("include_fixed=True is not supported when using ampl_repn") + if method is IncidenceMethod.ampl_repn and amplrepnvisitor is None: + # Developer error, this should never happen! + raise RuntimeError("_ampl_repn_visitor must be provided when using ampl_repn") + + # Dispatch to correct method if method is IncidenceMethod.identify_variables: return _get_incident_via_identify_variables(expr, include_fixed) elif method is IncidenceMethod.standard_repn: @@ -174,6 +183,5 @@ def get_incident_variables(expr, **kwds): else: raise ValueError( f"Unrecognized value {method} for the method used to identify incident" - f" variables. Valid options are {IncidenceMethod.identify_variables}" - f" and {IncidenceMethod.standard_repn}." + f" variables. See the IncidenceMethod enum for valid methods." ) diff --git a/pyomo/contrib/incidence_analysis/interface.py b/pyomo/contrib/incidence_analysis/interface.py index b8a6c1275f9..41f0ece3a75 100644 --- a/pyomo/contrib/incidence_analysis/interface.py +++ b/pyomo/contrib/incidence_analysis/interface.py @@ -29,7 +29,7 @@ plotly, ) from pyomo.common.deprecation import deprecated -from pyomo.contrib.incidence_analysis.config import IncidenceConfig, IncidenceMethod +from pyomo.contrib.incidence_analysis.config import get_config_from_kwds from pyomo.contrib.incidence_analysis.matching import maximum_matching from pyomo.contrib.incidence_analysis.connected import get_independent_submatrices from pyomo.contrib.incidence_analysis.triangularize import ( @@ -64,7 +64,7 @@ def _check_unindexed(complist): def get_incidence_graph(variables, constraints, **kwds): - config = IncidenceConfig(kwds) + config = get_config_from_kwds(**kwds) return get_bipartite_incidence_graph(variables, constraints, **config) @@ -95,7 +95,7 @@ def get_bipartite_incidence_graph(variables, constraints, **kwds): """ # Note that this ConfigDict contains the visitor that we will re-use # when constructing constraints. - config = IncidenceConfig(kwds) + config = get_config_from_kwds(**kwds) _check_unindexed(variables + constraints) N = len(variables) M = len(constraints) @@ -168,7 +168,7 @@ def extract_bipartite_subgraph(graph, nodes0, nodes1): def _generate_variables_in_constraints(constraints, **kwds): # Note: We construct a visitor here - config = IncidenceConfig(kwds) + config = get_config_from_kwds(**kwds) known_vars = ComponentSet() for con in constraints: for var in get_incident_variables(con.body, **config): @@ -196,7 +196,7 @@ def get_structural_incidence_matrix(variables, constraints, **kwds): Entries are 1.0. """ - config = IncidenceConfig(kwds) + config = get_config_from_kwds(**kwds) _check_unindexed(variables + constraints) N, M = len(variables), len(constraints) var_idx_map = ComponentMap((v, i) for i, v in enumerate(variables)) @@ -279,7 +279,7 @@ def __init__(self, model=None, active=True, include_inequality=True, **kwds): # to cache the incidence graph for fast analysis later on. # WARNING: This cache will become invalid if the user alters their # model. - self._config = IncidenceConfig(kwds) + self._config = get_config_from_kwds(**kwds) if model is None: self._incidence_graph = None self._variables = None From 57d3134725a82f150e4b63da2f32b93ade02d201 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Mon, 22 Jan 2024 15:11:22 -0700 Subject: [PATCH 065/103] remove now-unused ConfigDict hack --- pyomo/contrib/incidence_analysis/config.py | 39 ---------------------- 1 file changed, 39 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/config.py b/pyomo/contrib/incidence_analysis/config.py index 4ab086da508..d055be478fe 100644 --- a/pyomo/contrib/incidence_analysis/config.py +++ b/pyomo/contrib/incidence_analysis/config.py @@ -65,10 +65,6 @@ class IncidenceMethod(enum.Enum): ) -class _ReconstructVisitor: - pass - - def _amplrepnvisitor_validator(visitor): if not isinstance(visitor, AMPLRepnVisitor): raise TypeError( @@ -84,41 +80,6 @@ def _amplrepnvisitor_validator(visitor): ) -class _IncidenceConfigDict(ConfigDict): - def __call__( - self, - value=NOTSET, - default=NOTSET, - domain=NOTSET, - description=NOTSET, - doc=NOTSET, - visibility=NOTSET, - implicit=NOTSET, - implicit_domain=NOTSET, - preserve_implicit=False, - ): - init_value = value - new = super().__call__( - value=value, - default=default, - domain=domain, - description=description, - doc=doc, - visibility=visibility, - implicit=implicit, - implicit_domain=implicit_domain, - preserve_implicit=preserve_implicit, - ) - - if ( - new.method == IncidenceMethod.ampl_repn - and "_ampl_repn_visitor" not in init_value - ): - new._ampl_repn_visitor = _ReconstructVisitor - - return new - - IncidenceConfig = ConfigDict() """Options for incidence graph generation From f279fed36fe137a7d7bb33ebf48c4bd05d9f7619 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Mon, 22 Jan 2024 15:22:42 -0700 Subject: [PATCH 066/103] split imports onto separate lines --- pyomo/contrib/incidence_analysis/incidence.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/incidence_analysis/incidence.py b/pyomo/contrib/incidence_analysis/incidence.py index 1fc3380fe6b..636a400def4 100644 --- a/pyomo/contrib/incidence_analysis/incidence.py +++ b/pyomo/contrib/incidence_analysis/incidence.py @@ -20,7 +20,8 @@ from pyomo.repn.util import FileDeterminism, FileDeterminism_to_SortComponents from pyomo.util.subsystems import TemporarySubsystemManager from pyomo.contrib.incidence_analysis.config import ( - IncidenceMethod, get_config_from_kwds + IncidenceMethod, + get_config_from_kwds, ) From bac8bda15dea59856c2f03e4ffa33fb52afab4b9 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 31 Jan 2024 12:06:33 -0700 Subject: [PATCH 067/103] Correcting precedence and some other mistakes John caught --- pyomo/core/expr/logical_expr.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/pyomo/core/expr/logical_expr.py b/pyomo/core/expr/logical_expr.py index 31082293a71..aabef99597d 100644 --- a/pyomo/core/expr/logical_expr.py +++ b/pyomo/core/expr/logical_expr.py @@ -539,7 +539,7 @@ class AllDifferentExpression(NaryBooleanExpression): __slots__ = () - PRECEDENCE = 9 # TODO: maybe? + PRECEDENCE = None def getname(self, *arg, **kwd): return 'all_different' @@ -548,9 +548,13 @@ def _to_string(self, values, verbose, smap): return "all_different(%s)" % (", ".join(values)) def _apply_operation(self, result): - for val1, val2 in combinations(result, 2): - if val1 == val2: + last = None + # we know these are integer-valued, so we can just sort them an make + # sure that no adjacent pairs have the same value. + for val in sorted(result): + if last == val: return False + last = val return True @@ -561,13 +565,7 @@ class CountIfExpression(NumericExpression): """ __slots__ = () - PRECEDENCE = 10 # TODO: maybe? - - def __init__(self, args): - # require a list, a la SumExpression - if args.__class__ is not list: - args = list(args) - self._args_ = args + PRECEDENCE = None # NumericExpression assumes binary operator, so we have to override. def nargs(self): @@ -580,7 +578,7 @@ def _to_string(self, values, verbose, smap): return "count_if(%s)" % (", ".join(values)) def _apply_operation(self, result): - return sum(value(r) for r in result) + return sum(r for r in result) special_boolean_atom_types = {ExactlyExpression, AtMostExpression, AtLeastExpression} From 272ea35a03d305ccd4bffd6b5aa428a95b51d2c6 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 31 Jan 2024 12:56:11 -0700 Subject: [PATCH 068/103] Checking argument types for logical expressions --- pyomo/core/expr/logical_expr.py | 59 ++++++++++++++++--- .../tests/unit/test_logical_expr_expanded.py | 10 +++- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/pyomo/core/expr/logical_expr.py b/pyomo/core/expr/logical_expr.py index aabef99597d..d345e0f64ae 100644 --- a/pyomo/core/expr/logical_expr.py +++ b/pyomo/core/expr/logical_expr.py @@ -183,12 +183,57 @@ def _flattened(args): yield arg +def _flattened_boolean_args(args): + """Flatten any potentially indexed arguments and check that they are + Boolean-valued.""" + for arg in args: + if arg.__class__ in native_types: + myiter = (arg,) + elif isinstance(arg, (types.GeneratorType, list)): + myiter = arg + elif arg.is_indexed(): + myiter = arg.values() + else: + myiter = (arg,) + for _argdata in myiter: + if _argdata.__class__ in native_logical_types: + yield _argdata + elif hasattr(_argdata, 'is_logical_type') and _argdata.is_logical_type(): + yield _argdata + else: + raise ValueError( + "Non-Boolean-valued argument '%s' encountered when constructing " + "expression of Boolean arguments" % arg) + + +def _flattened_numeric_args(args): + """Flatten any potentially indexed arguments and check that they are + numeric.""" + for arg in args: + if arg.__class__ in native_types: + myiter = (arg,) + elif isinstance(arg, (types.GeneratorType, list)): + myiter = arg + elif arg.is_indexed(): + myiter = arg.values() + else: + myiter = (arg,) + for _argdata in myiter: + if _argdata.__class__ in native_numeric_types: + yield _argdata + elif hasattr(_argdata, 'is_numeric_type') and _argdata.is_numeric_type(): + yield _argdata + else: + raise ValueError( + "Non-numeric argument '%s' encountered when constructing " + "expression with numeric arguments" % arg) + def land(*args): """ Construct an AndExpression between passed arguments. """ result = AndExpression([]) - for argdata in _flattened(args): + for argdata in _flattened_boolean_args(args): result = result.add(argdata) return result @@ -198,7 +243,7 @@ def lor(*args): Construct an OrExpression between passed arguments. """ result = OrExpression([]) - for argdata in _flattened(args): + for argdata in _flattened_boolean_args(args): result = result.add(argdata) return result @@ -211,7 +256,7 @@ def exactly(n, *args): Usage: exactly(2, m.Y1, m.Y2, m.Y3, ...) """ - result = ExactlyExpression([n] + list(_flattened(args))) + result = ExactlyExpression([n] + list(_flattened_boolean_args(args))) return result @@ -223,7 +268,7 @@ def atmost(n, *args): Usage: atmost(2, m.Y1, m.Y2, m.Y3, ...) """ - result = AtMostExpression([n] + list(_flattened(args))) + result = AtMostExpression([n] + list(_flattened_boolean_args(args))) return result @@ -235,7 +280,7 @@ def atleast(n, *args): Usage: atleast(2, m.Y1, m.Y2, m.Y3, ...) """ - result = AtLeastExpression([n] + list(_flattened(args))) + result = AtLeastExpression([n] + list(_flattened_boolean_args(args))) return result @@ -246,7 +291,7 @@ def all_different(*args): Usage: all_different(m.X1, m.X2, ...) """ - return AllDifferentExpression(list(_flattened(args))) + return AllDifferentExpression(list(_flattened_numeric_args(args))) def count_if(*args): @@ -256,7 +301,7 @@ def count_if(*args): Usage: count_if(m.Y1, m.Y2, ...) """ - return CountIfExpression(list(_flattened(args))) + return CountIfExpression(list(_flattened_boolean_args(args))) class UnaryBooleanExpression(BooleanExpression): diff --git a/pyomo/core/tests/unit/test_logical_expr_expanded.py b/pyomo/core/tests/unit/test_logical_expr_expanded.py index ca2b64957ef..0e5bb4da445 100644 --- a/pyomo/core/tests/unit/test_logical_expr_expanded.py +++ b/pyomo/core/tests/unit/test_logical_expr_expanded.py @@ -280,6 +280,8 @@ def test_to_string(self): m.Y2 = BooleanVar() m.Y3 = BooleanVar() m.Y4 = BooleanVar() + m.int1 = Var(domain=Integers) + m.int2 = Var(domain=Integers) self.assertEqual(str(land(m.Y1, m.Y2, m.Y3)), "Y1 ∧ Y2 ∧ Y3") self.assertEqual(str(lor(m.Y1, m.Y2, m.Y3)), "Y1 ∨ Y2 ∨ Y3") @@ -289,7 +291,8 @@ def test_to_string(self): self.assertEqual(str(atleast(1, m.Y1, m.Y2)), "atleast(1: [Y1, Y2])") self.assertEqual(str(atmost(1, m.Y1, m.Y2)), "atmost(1: [Y1, Y2])") self.assertEqual(str(exactly(1, m.Y1, m.Y2)), "exactly(1: [Y1, Y2])") - self.assertEqual(str(all_different(m.Y1, m.Y2)), "all_different(Y1, Y2)") + self.assertEqual(str(all_different(m.int1, m.int2)), + "all_different(int1, int2)") self.assertEqual(str(count_if(m.Y1, m.Y2)), "count_if(Y1, Y2)") # Precedence checks @@ -308,12 +311,15 @@ def test_node_types(self): m.Y1 = BooleanVar() m.Y2 = BooleanVar() m.Y3 = BooleanVar() + m.int1 = Var(domain=Integers) + m.int2 = Var(domain=Integers) + m.int3 = Var(domain=Integers) self.assertFalse(m.Y1.is_expression_type()) self.assertTrue(lnot(m.Y1).is_expression_type()) self.assertTrue(equivalent(m.Y1, m.Y2).is_expression_type()) self.assertTrue(atmost(1, [m.Y1, m.Y2, m.Y3]).is_expression_type()) - self.assertTrue(all_different(m.Y1, m.Y2, m.Y3).is_expression_type()) + self.assertTrue(all_different(m.int1, m.int2, m.int3).is_expression_type()) self.assertTrue(count_if(m.Y1, m.Y2, m.Y3).is_expression_type()) def test_numeric_invalid(self): From e0edbe792caa3d7041abf496793b2c89fcd13e51 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 31 Jan 2024 12:57:24 -0700 Subject: [PATCH 069/103] Black disagrees --- pyomo/core/expr/logical_expr.py | 11 +++++++---- pyomo/core/tests/unit/test_logical_expr_expanded.py | 5 +++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pyomo/core/expr/logical_expr.py b/pyomo/core/expr/logical_expr.py index d345e0f64ae..17f4a4dd564 100644 --- a/pyomo/core/expr/logical_expr.py +++ b/pyomo/core/expr/logical_expr.py @@ -184,7 +184,7 @@ def _flattened(args): def _flattened_boolean_args(args): - """Flatten any potentially indexed arguments and check that they are + """Flatten any potentially indexed arguments and check that they are Boolean-valued.""" for arg in args: if arg.__class__ in native_types: @@ -203,11 +203,12 @@ def _flattened_boolean_args(args): else: raise ValueError( "Non-Boolean-valued argument '%s' encountered when constructing " - "expression of Boolean arguments" % arg) + "expression of Boolean arguments" % arg + ) def _flattened_numeric_args(args): - """Flatten any potentially indexed arguments and check that they are + """Flatten any potentially indexed arguments and check that they are numeric.""" for arg in args: if arg.__class__ in native_types: @@ -226,7 +227,9 @@ def _flattened_numeric_args(args): else: raise ValueError( "Non-numeric argument '%s' encountered when constructing " - "expression with numeric arguments" % arg) + "expression with numeric arguments" % arg + ) + def land(*args): """ diff --git a/pyomo/core/tests/unit/test_logical_expr_expanded.py b/pyomo/core/tests/unit/test_logical_expr_expanded.py index 0e5bb4da445..0360e9b4783 100644 --- a/pyomo/core/tests/unit/test_logical_expr_expanded.py +++ b/pyomo/core/tests/unit/test_logical_expr_expanded.py @@ -291,8 +291,9 @@ def test_to_string(self): self.assertEqual(str(atleast(1, m.Y1, m.Y2)), "atleast(1: [Y1, Y2])") self.assertEqual(str(atmost(1, m.Y1, m.Y2)), "atmost(1: [Y1, Y2])") self.assertEqual(str(exactly(1, m.Y1, m.Y2)), "exactly(1: [Y1, Y2])") - self.assertEqual(str(all_different(m.int1, m.int2)), - "all_different(int1, int2)") + self.assertEqual( + str(all_different(m.int1, m.int2)), "all_different(int1, int2)" + ) self.assertEqual(str(count_if(m.Y1, m.Y2)), "count_if(Y1, Y2)") # Precedence checks From cdc21268a71ec906e84dd3c8731d4c296abd6a01 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 31 Jan 2024 15:12:21 -0700 Subject: [PATCH 070/103] We do need to evaluate the args when we apply count_if because they can be relational expressions --- pyomo/core/expr/logical_expr.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyomo/core/expr/logical_expr.py b/pyomo/core/expr/logical_expr.py index 17f4a4dd564..875f5107f3a 100644 --- a/pyomo/core/expr/logical_expr.py +++ b/pyomo/core/expr/logical_expr.py @@ -200,6 +200,8 @@ def _flattened_boolean_args(args): yield _argdata elif hasattr(_argdata, 'is_logical_type') and _argdata.is_logical_type(): yield _argdata + elif isinstance(_argdata, BooleanValue): + yield _argdata else: raise ValueError( "Non-Boolean-valued argument '%s' encountered when constructing " @@ -626,7 +628,7 @@ def _to_string(self, values, verbose, smap): return "count_if(%s)" % (", ".join(values)) def _apply_operation(self, result): - return sum(r for r in result) + return sum(value(r) for r in result) special_boolean_atom_types = {ExactlyExpression, AtMostExpression, AtLeastExpression} From 8d2116265326a834680b9cc0bb896747d2749f78 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 31 Jan 2024 15:13:12 -0700 Subject: [PATCH 071/103] Removing another test with Boolean args to all diff --- pyomo/contrib/cp/tests/test_docplex_walker.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/pyomo/contrib/cp/tests/test_docplex_walker.py b/pyomo/contrib/cp/tests/test_docplex_walker.py index 0f1c73cd3b1..b897053c93a 100644 --- a/pyomo/contrib/cp/tests/test_docplex_walker.py +++ b/pyomo/contrib/cp/tests/test_docplex_walker.py @@ -424,22 +424,6 @@ def test_all_diff_expression(self): self.assertTrue(expr[1].equals(cp.all_diff(a[i] for i in m.I))) - def test_Boolean_args_in_all_diff_expression(self): - m = self.get_model() - m.a.domain = Integers - m.a.bounds = (11, 20) - m.c = LogicalConstraint(expr=all_different(m.a[1] == 13, m.b)) - - visitor = self.get_visitor() - expr = visitor.walk_expression((m.c.body, m.c, 0)) - - self.assertIn(id(m.a[1]), visitor.var_map) - a0 = visitor.var_map[id(m.a[1])] - self.assertIn(id(m.b), visitor.var_map) - b = visitor.var_map[id(m.b)] - - self.assertTrue(expr[1].equals(cp.all_diff(a0 == 13, b))) - def test_count_if_expression(self): m = self.get_model() m.a.domain = Integers From 1b73570e6cceab7127d0dca416dfad2774fedca6 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 31 Jan 2024 18:39:13 -0500 Subject: [PATCH 072/103] correct typos --- pyomo/contrib/mindtpy/algorithm_base_class.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index ad462221ec5..b6a223ba24b 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -108,7 +108,7 @@ def __init__(self, **kwds): self.curr_int_sol = [] self.should_terminate = False self.integer_list = [] - # Dictionary {integer solution (list): [cuts begin index, cuts end index] (list)} + # Dictionary {integer solution (tuple): [cuts begin index, cuts end index] (list)} self.integer_solution_to_cuts_index = dict() # Set up iteration counters @@ -2679,9 +2679,9 @@ def initialize_subsolvers(self): if config.mip_regularization_solver == 'gams': self.regularization_mip_opt.options['add_options'] = [] if config.regularization_mip_threads > 0: - self.regularization_mip_opt.options['threads'] = ( - config.regularization_mip_threads - ) + self.regularization_mip_opt.options[ + 'threads' + ] = config.regularization_mip_threads else: self.regularization_mip_opt.options['threads'] = config.threads @@ -2691,9 +2691,9 @@ def initialize_subsolvers(self): 'cplex_persistent', }: if config.solution_limit is not None: - self.regularization_mip_opt.options['mip_limits_solutions'] = ( - config.solution_limit - ) + self.regularization_mip_opt.options[ + 'mip_limits_solutions' + ] = config.solution_limit # We don't need to solve the regularization problem to optimality. # We will choose to perform aggressive node probing during presolve. self.regularization_mip_opt.options['mip_strategy_presolvenode'] = 3 @@ -2706,9 +2706,9 @@ def initialize_subsolvers(self): self.regularization_mip_opt.options['optimalitytarget'] = 3 elif config.mip_regularization_solver == 'gurobi': if config.solution_limit is not None: - self.regularization_mip_opt.options['SolutionLimit'] = ( - config.solution_limit - ) + self.regularization_mip_opt.options[ + 'SolutionLimit' + ] = config.solution_limit # Same reason as mip_strategy_presolvenode. self.regularization_mip_opt.options['Presolve'] = 2 @@ -3055,9 +3055,10 @@ def add_regularization(self): # The main problem might be unbounded, regularization is activated only when a valid bound is provided. if self.dual_bound != self.dual_bound_progress[0]: with time_code(self.timing, 'regularization main'): - (regularization_main_mip, regularization_main_mip_results) = ( - self.solve_regularization_main() - ) + ( + regularization_main_mip, + regularization_main_mip_results, + ) = self.solve_regularization_main() self.handle_regularization_main_tc( regularization_main_mip, regularization_main_mip_results ) From 4ec0e8c69ac7d1992f6879ba9b0e3e52352a9fea Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 31 Jan 2024 18:40:21 -0500 Subject: [PATCH 073/103] remove unused log --- pyomo/contrib/mindtpy/algorithm_base_class.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index b6a223ba24b..a4f3075a1e9 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -1290,7 +1290,6 @@ def handle_subproblem_infeasible(self, fixed_nlp, cb_opt=None): # elif var.has_lb() and abs(value(var) - var.lb) < config.absolute_bound_tolerance: # fixed_nlp.ipopt_zU_out[var] = -1 - # config.logger.info('Solving feasibility problem') feas_subproblem, feas_subproblem_results = self.solve_feasibility_subproblem() # TODO: do we really need this? if self.should_terminate: From acb10de438ccc8c7acb5b8ba7d42af7eab3694e9 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 31 Jan 2024 19:32:34 -0500 Subject: [PATCH 074/103] add one condition for fix dual bound --- pyomo/contrib/mindtpy/algorithm_base_class.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index a4f3075a1e9..ca428563a68 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -1561,7 +1561,7 @@ def fix_dual_bound(self, last_iter_cuts): self.handle_nlp_subproblem_tc(fixed_nlp, fixed_nlp_result) MindtPy = self.mip.MindtPy_utils - # deactivate the integer cuts generated after the best solution was found. + # Deactivate the integer cuts generated after the best solution was found. self.deactivate_no_good_cuts_when_fixing_bound(MindtPy.cuts.no_good_cuts) if ( config.add_regularization is not None @@ -3013,10 +3013,12 @@ def MindtPy_iteration_loop(self): # if add_no_good_cuts is True, the bound obtained in the last iteration is no reliable. # we correct it after the iteration. + # There is no need to fix the dual bound if no feasible solution has been found. if ( (config.add_no_good_cuts or config.use_tabu_list) and not self.should_terminate and config.add_regularization is None + and self.best_solution_found is not None ): self.fix_dual_bound(self.last_iter_cuts) config.logger.info( From de73340cf82b50d932ebe25a165c9bc3d7c63230 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 31 Jan 2024 19:41:24 -0500 Subject: [PATCH 075/103] remove fix_dual_bound for ECP method --- pyomo/contrib/mindtpy/extended_cutting_plane.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pyomo/contrib/mindtpy/extended_cutting_plane.py b/pyomo/contrib/mindtpy/extended_cutting_plane.py index ac13e352e35..0a98f88ed3f 100644 --- a/pyomo/contrib/mindtpy/extended_cutting_plane.py +++ b/pyomo/contrib/mindtpy/extended_cutting_plane.py @@ -66,12 +66,6 @@ def MindtPy_iteration_loop(self): add_ecp_cuts(self.mip, self.jacobians, self.config, self.timing) - # if add_no_good_cuts is True, the bound obtained in the last iteration is no reliable. - # we correct it after the iteration. - if ( - self.config.add_no_good_cuts or self.config.use_tabu_list - ) and not self.should_terminate: - self.fix_dual_bound(self.last_iter_cuts) self.config.logger.info( ' ===============================================================================================' ) From 92d9477985e92bcccd2785c4862eb1491dca3488 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Wed, 31 Jan 2024 19:52:56 -0500 Subject: [PATCH 076/103] black format --- pyomo/contrib/mindtpy/single_tree.py | 7 ++++--- pyomo/contrib/mindtpy/tests/nonconvex3.py | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index c1e52ed72d3..228810a8f90 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -588,9 +588,10 @@ def handle_lazy_subproblem_infeasible(self, fixed_nlp, mindtpy_solver, config, o dual_values = None config.logger.info('Solving feasibility problem') - (feas_subproblem, feas_subproblem_results) = ( - mindtpy_solver.solve_feasibility_subproblem() - ) + ( + feas_subproblem, + feas_subproblem_results, + ) = mindtpy_solver.solve_feasibility_subproblem() # In OA algorithm, OA cuts are generated based on the solution of the subproblem # We need to first copy the value of variables from the subproblem and then add cuts copy_var_list_values( diff --git a/pyomo/contrib/mindtpy/tests/nonconvex3.py b/pyomo/contrib/mindtpy/tests/nonconvex3.py index b08deb67b63..dbb88bb1fad 100644 --- a/pyomo/contrib/mindtpy/tests/nonconvex3.py +++ b/pyomo/contrib/mindtpy/tests/nonconvex3.py @@ -40,7 +40,9 @@ def __init__(self, *args, **kwargs): m.objective = Objective(expr=7 * m.x1 + 10 * m.x2, sense=minimize) - m.c1 = Constraint(expr=(m.x1**1.2) * (m.x2**1.7) - 7 * m.x1 - 9 * m.x2 <= -24) + m.c1 = Constraint( + expr=(m.x1**1.2) * (m.x2**1.7) - 7 * m.x1 - 9 * m.x2 <= -24 + ) m.c2 = Constraint(expr=-m.x1 - 2 * m.x2 <= 5) m.c3 = Constraint(expr=-3 * m.x1 + m.x2 <= 1) m.c4 = Constraint(expr=4 * m.x1 - 3 * m.x2 <= 11) From 42688d12649e915d1213739e9a93c6e9fce60062 Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Thu, 1 Feb 2024 09:23:58 -0700 Subject: [PATCH 077/103] Actions Version Update: Address node.js deprecations --- .github/workflows/release_wheel_creation.yml | 2 +- .github/workflows/test_branches.yml | 20 ++++++++++---------- .github/workflows/test_pr_and_main.yml | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release_wheel_creation.yml b/.github/workflows/release_wheel_creation.yml index 72a3ce1110b..3c837cb62b2 100644 --- a/.github/workflows/release_wheel_creation.yml +++ b/.github/workflows/release_wheel_creation.yml @@ -91,7 +91,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index a55a5db2433..1a1a0e5bd57 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -35,7 +35,7 @@ jobs: - name: Checkout Pyomo source uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10' - name: Black Formatting Check @@ -134,7 +134,7 @@ jobs: | tr '\n' ' ' | sed 's/ \+/ /g' >> $GITHUB_ENV #- name: Pip package cache - # uses: actions/cache@v3 + # uses: actions/cache@v4 # if: matrix.PYENV == 'pip' # id: pip-cache # with: @@ -142,7 +142,7 @@ jobs: # key: pip-${{env.CACHE_VER}}.0-${{runner.os}}-${{matrix.python}} #- name: OS package cache - # uses: actions/cache@v3 + # uses: actions/cache@v4 # if: matrix.TARGET != 'osx' # id: os-cache # with: @@ -150,7 +150,7 @@ jobs: # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} - name: TPL package download cache - uses: actions/cache@v3 + uses: actions/cache@v4 if: ${{ ! matrix.slim }} id: download-cache with: @@ -202,13 +202,13 @@ jobs: - name: Set up Python ${{ matrix.python }} if: matrix.PYENV == 'pip' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Set up Miniconda Python ${{ matrix.python }} if: matrix.PYENV == 'conda' - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: false python-version: ${{ matrix.python }} @@ -668,7 +668,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 @@ -724,19 +724,19 @@ jobs: # We need the source for .codecov.yml and running "coverage xml" #- name: Pip package cache - # uses: actions/cache@v3 + # uses: actions/cache@v4 # id: pip-cache # with: # path: cache/pip # key: pip-${{env.CACHE_VER}}.0-${{runner.os}}-3.8 - name: Download build artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: artifacts - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index ac7691d32ae..f9a8cef99b2 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -38,7 +38,7 @@ jobs: - name: Checkout Pyomo source uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10' - name: Black Formatting Check @@ -164,7 +164,7 @@ jobs: | tr '\n' ' ' | sed 's/ \+/ /g' >> $GITHUB_ENV #- name: Pip package cache - # uses: actions/cache@v3 + # uses: actions/cache@v4 # if: matrix.PYENV == 'pip' # id: pip-cache # with: @@ -172,7 +172,7 @@ jobs: # key: pip-${{env.CACHE_VER}}.0-${{runner.os}}-${{matrix.python}} #- name: OS package cache - # uses: actions/cache@v3 + # uses: actions/cache@v4 # if: matrix.TARGET != 'osx' # id: os-cache # with: @@ -180,7 +180,7 @@ jobs: # key: pkg-${{env.CACHE_VER}}.0-${{runner.os}} - name: TPL package download cache - uses: actions/cache@v3 + uses: actions/cache@v4 if: ${{ ! matrix.slim }} id: download-cache with: @@ -232,13 +232,13 @@ jobs: - name: Set up Python ${{ matrix.python }} if: matrix.PYENV == 'pip' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Set up Miniconda Python ${{ matrix.python }} if: matrix.PYENV == 'conda' - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: false python-version: ${{ matrix.python }} @@ -699,7 +699,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 @@ -755,19 +755,19 @@ jobs: # We need the source for .codecov.yml and running "coverage xml" #- name: Pip package cache - # uses: actions/cache@v3 + # uses: actions/cache@v4 # id: pip-cache # with: # path: cache/pip # key: pip-${{env.CACHE_VER}}.0-${{runner.os}}-3.8 - name: Download build artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: artifacts - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 From 8d051b09ebaf84ec946b2d2e4f64a865bedde66e Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Thu, 1 Feb 2024 13:06:34 -0500 Subject: [PATCH 078/103] black format --- pyomo/contrib/mindtpy/algorithm_base_class.py | 25 +++++++++---------- pyomo/contrib/mindtpy/single_tree.py | 7 +++--- pyomo/contrib/mindtpy/tests/nonconvex3.py | 4 +-- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/pyomo/contrib/mindtpy/algorithm_base_class.py b/pyomo/contrib/mindtpy/algorithm_base_class.py index ca428563a68..3d5a7ebad03 100644 --- a/pyomo/contrib/mindtpy/algorithm_base_class.py +++ b/pyomo/contrib/mindtpy/algorithm_base_class.py @@ -2678,9 +2678,9 @@ def initialize_subsolvers(self): if config.mip_regularization_solver == 'gams': self.regularization_mip_opt.options['add_options'] = [] if config.regularization_mip_threads > 0: - self.regularization_mip_opt.options[ - 'threads' - ] = config.regularization_mip_threads + self.regularization_mip_opt.options['threads'] = ( + config.regularization_mip_threads + ) else: self.regularization_mip_opt.options['threads'] = config.threads @@ -2690,9 +2690,9 @@ def initialize_subsolvers(self): 'cplex_persistent', }: if config.solution_limit is not None: - self.regularization_mip_opt.options[ - 'mip_limits_solutions' - ] = config.solution_limit + self.regularization_mip_opt.options['mip_limits_solutions'] = ( + config.solution_limit + ) # We don't need to solve the regularization problem to optimality. # We will choose to perform aggressive node probing during presolve. self.regularization_mip_opt.options['mip_strategy_presolvenode'] = 3 @@ -2705,9 +2705,9 @@ def initialize_subsolvers(self): self.regularization_mip_opt.options['optimalitytarget'] = 3 elif config.mip_regularization_solver == 'gurobi': if config.solution_limit is not None: - self.regularization_mip_opt.options[ - 'SolutionLimit' - ] = config.solution_limit + self.regularization_mip_opt.options['SolutionLimit'] = ( + config.solution_limit + ) # Same reason as mip_strategy_presolvenode. self.regularization_mip_opt.options['Presolve'] = 2 @@ -3056,10 +3056,9 @@ def add_regularization(self): # The main problem might be unbounded, regularization is activated only when a valid bound is provided. if self.dual_bound != self.dual_bound_progress[0]: with time_code(self.timing, 'regularization main'): - ( - regularization_main_mip, - regularization_main_mip_results, - ) = self.solve_regularization_main() + (regularization_main_mip, regularization_main_mip_results) = ( + self.solve_regularization_main() + ) self.handle_regularization_main_tc( regularization_main_mip, regularization_main_mip_results ) diff --git a/pyomo/contrib/mindtpy/single_tree.py b/pyomo/contrib/mindtpy/single_tree.py index 228810a8f90..c1e52ed72d3 100644 --- a/pyomo/contrib/mindtpy/single_tree.py +++ b/pyomo/contrib/mindtpy/single_tree.py @@ -588,10 +588,9 @@ def handle_lazy_subproblem_infeasible(self, fixed_nlp, mindtpy_solver, config, o dual_values = None config.logger.info('Solving feasibility problem') - ( - feas_subproblem, - feas_subproblem_results, - ) = mindtpy_solver.solve_feasibility_subproblem() + (feas_subproblem, feas_subproblem_results) = ( + mindtpy_solver.solve_feasibility_subproblem() + ) # In OA algorithm, OA cuts are generated based on the solution of the subproblem # We need to first copy the value of variables from the subproblem and then add cuts copy_var_list_values( diff --git a/pyomo/contrib/mindtpy/tests/nonconvex3.py b/pyomo/contrib/mindtpy/tests/nonconvex3.py index dbb88bb1fad..b08deb67b63 100644 --- a/pyomo/contrib/mindtpy/tests/nonconvex3.py +++ b/pyomo/contrib/mindtpy/tests/nonconvex3.py @@ -40,9 +40,7 @@ def __init__(self, *args, **kwargs): m.objective = Objective(expr=7 * m.x1 + 10 * m.x2, sense=minimize) - m.c1 = Constraint( - expr=(m.x1**1.2) * (m.x2**1.7) - 7 * m.x1 - 9 * m.x2 <= -24 - ) + m.c1 = Constraint(expr=(m.x1**1.2) * (m.x2**1.7) - 7 * m.x1 - 9 * m.x2 <= -24) m.c2 = Constraint(expr=-m.x1 - 2 * m.x2 <= 5) m.c3 = Constraint(expr=-3 * m.x1 + m.x2 <= 1) m.c4 = Constraint(expr=4 * m.x1 - 3 * m.x2 <= 11) From 71a0a09921709a06dc3da394a2c697616abda2bf Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Thu, 1 Feb 2024 11:29:59 -0700 Subject: [PATCH 079/103] Update upload-artifact version --- .github/workflows/release_wheel_creation.yml | 6 +++--- .github/workflows/test_branches.yml | 2 +- .github/workflows/test_pr_and_main.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release_wheel_creation.yml b/.github/workflows/release_wheel_creation.yml index 3c837cb62b2..ef44806d6d4 100644 --- a/.github/workflows/release_wheel_creation.yml +++ b/.github/workflows/release_wheel_creation.yml @@ -41,7 +41,7 @@ jobs: CIBW_BUILD_VERBOSITY: 1 CIBW_BEFORE_BUILD: pip install cython pybind11 CIBW_CONFIG_SETTINGS: '--global-option="--with-cython --with-distributable-extensions"' - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: native_wheels path: dist/*.whl @@ -72,7 +72,7 @@ jobs: CIBW_BUILD_VERBOSITY: 1 CIBW_BEFORE_BUILD: pip install cython pybind11 CIBW_CONFIG_SETTINGS: '--global-option="--with-cython --with-distributable-extensions"' - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: alt_wheels path: dist/*.whl @@ -102,7 +102,7 @@ jobs: run: | python setup.py --without-cython sdist --format=gztar - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: generictarball path: dist diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 1a1a0e5bd57..a6857fb996e 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -648,7 +648,7 @@ jobs: coverage xml -i - name: Record build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{github.job}}_${{env.GHA_JOBGROUP}}-${{env.GHA_JOBNAME}} path: | diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index f9a8cef99b2..949d3099d74 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -678,7 +678,7 @@ jobs: coverage xml -i - name: Record build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{github.job}}_${{env.GHA_JOBGROUP}}-${{env.GHA_JOBNAME}} path: | From 1d243baef3301fb7c552d701adaa85b8b8bfaf4d Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Thu, 1 Feb 2024 12:23:40 -0700 Subject: [PATCH 080/103] Update codecov-action version --- .github/workflows/test_branches.yml | 4 ++-- .github/workflows/test_pr_and_main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index a6857fb996e..e5513d25975 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -831,7 +831,7 @@ jobs: - name: Upload codecov reports if: github.repository_owner == 'Pyomo' || github.ref != 'refs/heads/main' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: coverage.xml token: ${{ secrets.PYOMO_CODECOV_TOKEN }} @@ -843,7 +843,7 @@ jobs: if: | hashFiles('coverage-other.xml') != '' && (github.repository_owner == 'Pyomo' || github.ref != 'refs/heads/main') - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: coverage-other.xml token: ${{ secrets.PYOMO_CODECOV_TOKEN }} diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 949d3099d74..c5028606c17 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -862,7 +862,7 @@ jobs: - name: Upload codecov reports if: github.repository_owner == 'Pyomo' || github.ref != 'refs/heads/main' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: coverage.xml token: ${{ secrets.PYOMO_CODECOV_TOKEN }} @@ -874,7 +874,7 @@ jobs: if: | hashFiles('coverage-other.xml') != '' && (github.repository_owner == 'Pyomo' || github.ref != 'refs/heads/main') - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: coverage-other.xml token: ${{ secrets.PYOMO_CODECOV_TOKEN }} From ff0d0351b687d95678181f07f401f5deb8ee6f17 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 1 Feb 2024 15:15:03 -0700 Subject: [PATCH 081/103] Fix RangeSet.__len__ when defined by floats --- pyomo/core/base/set.py | 2 +- pyomo/core/tests/unit/test_set.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index ba7fdd52446..d820ae8d933 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -2668,7 +2668,7 @@ def __len__(self): if r.start == r.end: return 1 else: - return (r.end - r.start) // r.step + 1 + return int((r.end - r.start) // r.step) + 1 else: return sum(1 for _ in self) diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index a1072e7156c..2154c02e659 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -1238,6 +1238,9 @@ def __len__(self): # Test types that cannot be case to set self.assertNotEqual(SetOf({3}), 3) + # Test floats + self.assertEqual(RangeSet(0.0, 2.0), RangeSet(0.0, 2.0)) + def test_inequality(self): self.assertTrue(SetOf([1, 2, 3]) <= SetOf({1, 2, 3})) self.assertFalse(SetOf([1, 2, 3]) < SetOf({1, 2, 3})) From 714edcd069677c9d1ac09a67f9e10e4dc2871ff4 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 1 Feb 2024 15:22:06 -0700 Subject: [PATCH 082/103] Expand the test to mixed-type RangeSets --- pyomo/core/tests/unit/test_set.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index 2154c02e659..72231bb08d7 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -1240,6 +1240,7 @@ def __len__(self): # Test floats self.assertEqual(RangeSet(0.0, 2.0), RangeSet(0.0, 2.0)) + self.assertEqual(RangeSet(0.0, 2.0), RangeSet(0, 2)) def test_inequality(self): self.assertTrue(SetOf([1, 2, 3]) <= SetOf({1, 2, 3})) From 3760de2e36254457962d06b53d92046958ffa911 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 5 Feb 2024 11:39:45 -0700 Subject: [PATCH 083/103] Remove unneeded import --- pyomo/core/expr/logical_expr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/core/expr/logical_expr.py b/pyomo/core/expr/logical_expr.py index 875f5107f3a..48daa79a5b3 100644 --- a/pyomo/core/expr/logical_expr.py +++ b/pyomo/core/expr/logical_expr.py @@ -11,7 +11,7 @@ # ___________________________________________________________________________ import types -from itertools import combinations, islice +from itertools import islice import logging import traceback From 6f1a0552388f25727563908abde9f1b405b6e4b0 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 7 Feb 2024 15:39:39 -0700 Subject: [PATCH 084/103] Update ExitNodeDispatcher to be compatible with inherited expression types --- pyomo/repn/util.py | 67 +++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/pyomo/repn/util.py b/pyomo/repn/util.py index b65aa9427d5..108bb0ab972 100644 --- a/pyomo/repn/util.py +++ b/pyomo/repn/util.py @@ -387,42 +387,49 @@ def __init__(self, *args, **kwargs): super().__init__(None, *args, **kwargs) def __missing__(self, key): - return functools.partial(self.register_dispatcher, key=key) - - def register_dispatcher(self, visitor, node, *data, key=None): + if type(key) is tuple: + node_class = key[0] + else: + node_class = key + bases = node_class.__mro__ + # Note: if we add an `etype`, then this special-case can be removed if ( - isinstance(node, _named_subexpression_types) - or type(node) is kernel.expression.noclone + issubclass(node_class, _named_subexpression_types) + or node_class is kernel.expression.noclone ): - base_type = Expression - elif not node.is_potentially_variable(): - base_type = node.potentially_variable_base_class() - else: - base_type = node.__class__ - if isinstance(key, tuple): - base_key = (base_type,) + key[1:] - # Only cache handlers for unary, binary and ternary operators - cache = len(key) <= 4 - else: - base_key = base_type - cache = True - if base_key in self: - fcn = self[base_key] - elif base_type in self: - fcn = self[base_type] - elif any((k[0] if k.__class__ is tuple else k) is base_type for k in self): - raise DeveloperError( - f"Base expression key '{base_key}' not found when inserting dispatcher" - f" for node '{type(node).__name__}' while walking expression tree." - ) - else: + bases = [Expression] + fcn = None + for base_type in bases: + if isinstance(key, tuple): + base_key = (base_type,) + key[1:] + # Only cache handlers for unary, binary and ternary operators + cache = len(key) <= 4 + else: + base_key = base_type + cache = True + if base_key in self: + fcn = self[base_key] + elif base_type in self: + fcn = self[base_type] + elif any((k[0] if type(k) is tuple else k) is base_type for k in self): + raise DeveloperError( + f"Base expression key '{base_key}' not found when inserting " + f"dispatcher for node '{node_class.__name__}' while walking " + "expression tree." + ) + if fcn is None: + if type(key) is tuple: + node_class = key[0] + else: + node_class = key raise DeveloperError( - f"Unexpected expression node type '{type(node).__name__}' " - "found while walking expression tree." + f"Unexpected expression node type '{node_class.__name__}' " + f"found while walking expression tree." ) + return self.unexpected_expression_type(key) if cache: self[key] = fcn - return fcn(visitor, node, *data) + return fcn def apply_node_operation(node, args): From 51d23370f198a98481d0260c4b72f93b757c9406 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 7 Feb 2024 15:40:24 -0700 Subject: [PATCH 085/103] Refactor ExitNodeDispatcher to provide hook for unknown classes --- pyomo/repn/util.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pyomo/repn/util.py b/pyomo/repn/util.py index 108bb0ab972..cb67dd92494 100644 --- a/pyomo/repn/util.py +++ b/pyomo/repn/util.py @@ -418,19 +418,21 @@ def __missing__(self, key): "expression tree." ) if fcn is None: - if type(key) is tuple: - node_class = key[0] - else: - node_class = key - raise DeveloperError( - f"Unexpected expression node type '{node_class.__name__}' " - f"found while walking expression tree." - ) return self.unexpected_expression_type(key) if cache: self[key] = fcn return fcn + def unexpected_expression_type(self, key): + if type(key) is tuple: + node_class = key[0] + else: + node_class = key + raise DeveloperError( + f"Unexpected expression node type '{node_class.__name__}' " + f"found while walking expression tree." + ) + def apply_node_operation(node, args): try: From ce7a6b54256f03c24a1089223d355003869bfa63 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 7 Feb 2024 15:40:39 -0700 Subject: [PATCH 086/103] Add tests for inherited classes --- pyomo/repn/tests/test_util.py | 36 +++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/pyomo/repn/tests/test_util.py b/pyomo/repn/tests/test_util.py index 47cc6b1a63a..3f455aad13f 100644 --- a/pyomo/repn/tests/test_util.py +++ b/pyomo/repn/tests/test_util.py @@ -19,6 +19,7 @@ from pyomo.common.errors import DeveloperError, InvalidValueError from pyomo.common.log import LoggingIntercept from pyomo.core.expr import ( + NumericExpression, ProductExpression, NPV_ProductExpression, SumExpression, @@ -671,16 +672,6 @@ def test_ExitNodeDispatcher_registration(self): self.assertEqual(len(end), 4) self.assertIn(NPV_ProductExpression, end) - class NewProductExpression(ProductExpression): - pass - - node = NewProductExpression((6, 7)) - with self.assertRaisesRegex( - DeveloperError, r".*Unexpected expression node type 'NewProductExpression'" - ): - end[node.__class__](None, node, *node.args) - self.assertEqual(len(end), 4) - end[SumExpression, 2] = lambda v, n, *d: 2 * sum(d) self.assertEqual(len(end), 5) @@ -710,6 +701,31 @@ class NewProductExpression(ProductExpression): self.assertEqual(len(end), 7) self.assertNotIn((SumExpression, 3, 4, 5, 6), end) + class NewProductExpression(ProductExpression): + pass + + node = NewProductExpression((6, 7)) + self.assertEqual(end[node.__class__](None, node, *node.args), 42) + self.assertEqual(len(end), 8) + self.assertIn(NewProductExpression, end) + + class UnknownExpression(NumericExpression): + pass + + node = UnknownExpression((6, 7)) + with self.assertRaisesRegex( + DeveloperError, r".*Unexpected expression node type 'UnknownExpression'" + ): + end[node.__class__](None, node, *node.args) + self.assertEqual(len(end), 8) + + node = UnknownExpression((6, 7)) + with self.assertRaisesRegex( + DeveloperError, r".*Unexpected expression node type 'UnknownExpression'" + ): + end[node.__class__, 6, 7](None, node, *node.args) + self.assertEqual(len(end), 8) + def test_BeforeChildDispatcher_registration(self): class BeforeChildDispatcherTester(BeforeChildDispatcher): @staticmethod From db3b3abc132b6c45545248cca4a4b48cc8d22214 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 8 Feb 2024 15:38:22 -0700 Subject: [PATCH 087/103] Reorganizing the dispatcher structure to better support nested types --- .../contrib/fbbt/expression_bounds_walker.py | 37 +++++++++++-------- .../tests/test_expression_bounds_walker.py | 14 +++++-- pyomo/core/expr/__init__.py | 1 + pyomo/repn/tests/test_linear.py | 2 +- pyomo/repn/tests/test_util.py | 4 +- pyomo/repn/util.py | 34 ++++++++++------- 6 files changed, 57 insertions(+), 35 deletions(-) diff --git a/pyomo/contrib/fbbt/expression_bounds_walker.py b/pyomo/contrib/fbbt/expression_bounds_walker.py index 426d30f0ee6..22ebf28ae81 100644 --- a/pyomo/contrib/fbbt/expression_bounds_walker.py +++ b/pyomo/contrib/fbbt/expression_bounds_walker.py @@ -60,6 +60,14 @@ def _before_external_function(visitor, child): # this then this should use them return False, (-inf, inf) + @staticmethod + def _before_native_numeric(visitor, child): + return False, (child, child) + + @staticmethod + def _before_native_logical(visitor, child): + return False, (Bool(child), Bool(child)) + @staticmethod def _before_var(visitor, child): leaf_bounds = visitor.leaf_bounds @@ -67,12 +75,15 @@ def _before_var(visitor, child): pass elif child.is_fixed() and visitor.use_fixed_var_values_as_bounds: val = child.value - if val is None: + try: + ans = visitor._before_child_handlers[val.__class__](visitor, val) + except ValueError: raise ValueError( "Var '%s' is fixed to None. This value cannot be used to " "calculate bounds." % child.name - ) - leaf_bounds[child] = (child.value, child.value) + ) from None + leaf_bounds[child] = ans[1] + return ans else: lb = child.lb ub = child.ub @@ -93,23 +104,20 @@ def _before_named_expression(visitor, child): @staticmethod def _before_param(visitor, child): - return False, (child.value, child.value) - - @staticmethod - def _before_native(visitor, child): - return False, (child, child) + val = child.value + return visitor._before_child_handlers[val.__class__](visitor, val) @staticmethod def _before_string(visitor, child): raise ValueError( - f"{child!r} ({type(child)}) is not a valid numeric type. " + f"{child!r} ({type(child).__name__}) is not a valid numeric type. " f"Cannot compute bounds on expression." ) @staticmethod def _before_invalid(visitor, child): raise ValueError( - f"{child!r} ({type(child)}) is not a valid numeric type. " + f"{child!r} ({type(child).__name__}) is not a valid numeric type. " f"Cannot compute bounds on expression." ) @@ -123,10 +131,7 @@ def _before_complex(visitor, child): @staticmethod def _before_npv(visitor, child): val = value(child) - return False, (val, val) - - -_before_child_handlers = ExpressionBoundsBeforeChildDispatcher() + return visitor._before_child_handlers[val.__class__](visitor, val) def _handle_ProductExpression(visitor, node, arg1, arg2): @@ -277,7 +282,7 @@ def initializeWalker(self, expr): return True, expr def beforeChild(self, node, child, child_idx): - return _before_child_handlers[child.__class__](self, child) + return self._before_child_handlers[child.__class__](self, child) def exitNode(self, node, data): - return _operator_dispatcher[node.__class__](self, node, *data) + return self._operator_dispatcher[node.__class__](self, node, *data) diff --git a/pyomo/contrib/fbbt/tests/test_expression_bounds_walker.py b/pyomo/contrib/fbbt/tests/test_expression_bounds_walker.py index c51230155a7..612a3101ef7 100644 --- a/pyomo/contrib/fbbt/tests/test_expression_bounds_walker.py +++ b/pyomo/contrib/fbbt/tests/test_expression_bounds_walker.py @@ -273,11 +273,19 @@ def test_npv_expression(self): def test_invalid_numeric_type(self): m = self.make_model() - m.p = Param(initialize=True, domain=Any) + m.p = Param(initialize=True, mutable=True, domain=Any) visitor = ExpressionBoundsVisitor() with self.assertRaisesRegex( ValueError, - r"True \(\) is not a valid numeric type. " + r"True \(bool\) is not a valid numeric type. " + r"Cannot compute bounds on expression.", + ): + lb, ub = visitor.walk_expression(m.p + m.y) + + m.p.set_value(None) + with self.assertRaisesRegex( + ValueError, + r"None \(NoneType\) is not a valid numeric type. " r"Cannot compute bounds on expression.", ): lb, ub = visitor.walk_expression(m.p + m.y) @@ -288,7 +296,7 @@ def test_invalid_string(self): visitor = ExpressionBoundsVisitor() with self.assertRaisesRegex( ValueError, - r"'True' \(\) is not a valid numeric type. " + r"'True' \(str\) is not a valid numeric type. " r"Cannot compute bounds on expression.", ): lb, ub = visitor.walk_expression(m.p + m.y) diff --git a/pyomo/core/expr/__init__.py b/pyomo/core/expr/__init__.py index bd6d1b995a1..a03578de957 100644 --- a/pyomo/core/expr/__init__.py +++ b/pyomo/core/expr/__init__.py @@ -56,6 +56,7 @@ # BooleanValue, BooleanConstant, + BooleanExpression, BooleanExpressionBase, # UnaryBooleanExpression, diff --git a/pyomo/repn/tests/test_linear.py b/pyomo/repn/tests/test_linear.py index 0eec8a1541c..d4f268ae182 100644 --- a/pyomo/repn/tests/test_linear.py +++ b/pyomo/repn/tests/test_linear.py @@ -1517,7 +1517,7 @@ def test_type_registrations(self): bcd.register_dispatcher(visitor, 5), (False, (linear._CONSTANT, 5)) ) self.assertEqual(len(bcd), 1) - self.assertIs(bcd[int], bcd._before_native) + self.assertIs(bcd[int], bcd._before_native_numeric) # complex type self.assertEqual( bcd.register_dispatcher(visitor, 5j), (False, (linear._CONSTANT, 5j)) diff --git a/pyomo/repn/tests/test_util.py b/pyomo/repn/tests/test_util.py index 3f455aad13f..8ea6bda83b9 100644 --- a/pyomo/repn/tests/test_util.py +++ b/pyomo/repn/tests/test_util.py @@ -750,7 +750,7 @@ def evaluate(self, node): node = 5 self.assertEqual(bcd[node.__class__](None, node), (False, (_CONSTANT, 5))) - self.assertIs(bcd[int], bcd._before_native) + self.assertIs(bcd[int], bcd._before_native_numeric) self.assertEqual(len(bcd), 1) node = 'string' @@ -787,7 +787,7 @@ class new_int(int): node = new_int(5) self.assertEqual(bcd[node.__class__](None, node), (False, (_CONSTANT, 5))) - self.assertIs(bcd[new_int], bcd._before_native) + self.assertIs(bcd[new_int], bcd._before_native_numeric) self.assertEqual(len(bcd), 5) node = [] diff --git a/pyomo/repn/util.py b/pyomo/repn/util.py index cb67dd92494..634b4d1d640 100644 --- a/pyomo/repn/util.py +++ b/pyomo/repn/util.py @@ -25,6 +25,7 @@ native_types, native_numeric_types, native_complex_types, + native_logical_types, ) from pyomo.core.pyomoobject import PyomoObject from pyomo.core.base import ( @@ -265,7 +266,9 @@ def __missing__(self, key): def register_dispatcher(self, visitor, child): child_type = type(child) if child_type in native_numeric_types: - self[child_type] = self._before_native + self[child_type] = self._before_native_numeric + elif child_type in native_logical_types: + self[child_type] = self._before_native_logical elif issubclass(child_type, str): self[child_type] = self._before_string elif child_type in native_types: @@ -275,7 +278,7 @@ def register_dispatcher(self, visitor, child): self[child_type] = self._before_invalid elif not hasattr(child, 'is_expression_type'): if check_if_numeric_type(child): - self[child_type] = self._before_native + self[child_type] = self._before_native_numeric else: self[child_type] = self._before_invalid elif not child.is_expression_type(): @@ -306,9 +309,18 @@ def _before_general_expression(visitor, child): return True, None @staticmethod - def _before_native(visitor, child): + def _before_native_numeric(visitor, child): return False, (_CONSTANT, child) + @staticmethod + def _before_native_logical(visitor, child): + return False, ( + _CONSTANT, + InvalidNumber( + child, f"{child!r} ({type(child).__name__}) is not a valid numeric type" + ), + ) + @staticmethod def _before_complex(visitor, child): return False, (_CONSTANT, complex_number_error(child, visitor, child)) @@ -318,7 +330,7 @@ def _before_invalid(visitor, child): return False, ( _CONSTANT, InvalidNumber( - child, f"{child!r} ({type(child)}) is not a valid numeric type" + child, f"{child!r} ({type(child).__name__}) is not a valid numeric type" ), ) @@ -327,7 +339,7 @@ def _before_string(visitor, child): return False, ( _CONSTANT, InvalidNumber( - child, f"{child!r} ({type(child)}) is not a valid numeric type" + child, f"{child!r} ({type(child).__name__}) is not a valid numeric type" ), ) @@ -418,19 +430,15 @@ def __missing__(self, key): "expression tree." ) if fcn is None: - return self.unexpected_expression_type(key) + fcn = self.unexpected_expression_type if cache: self[key] = fcn return fcn - def unexpected_expression_type(self, key): - if type(key) is tuple: - node_class = key[0] - else: - node_class = key + def unexpected_expression_type(self, visitor, node, *arg): raise DeveloperError( - f"Unexpected expression node type '{node_class.__name__}' " - f"found while walking expression tree." + f"Unexpected expression node type '{type(node).__name__}' " + f"found while walking expression tree in {type(visitor).__name__}." ) From 3a9fc9fda5d567a8af40de383a84352050bdc687 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 8 Feb 2024 15:39:43 -0700 Subject: [PATCH 088/103] Additional dispatcher restructuring --- .../contrib/fbbt/expression_bounds_walker.py | 59 ++++++++++++++----- .../tests/test_expression_bounds_walker.py | 29 ++++++++- 2 files changed, 71 insertions(+), 17 deletions(-) diff --git a/pyomo/contrib/fbbt/expression_bounds_walker.py b/pyomo/contrib/fbbt/expression_bounds_walker.py index 22ebf28ae81..31e52e0dc79 100644 --- a/pyomo/contrib/fbbt/expression_bounds_walker.py +++ b/pyomo/contrib/fbbt/expression_bounds_walker.py @@ -9,6 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import logging from math import pi from pyomo.common.collections import ComponentMap from pyomo.contrib.fbbt.interval import ( @@ -30,6 +31,7 @@ ) from pyomo.core.base.expression import Expression from pyomo.core.expr.numeric_expr import ( + NumericExpression, NegationExpression, ProductExpression, DivisionExpression, @@ -40,12 +42,20 @@ LinearExpression, SumExpression, ExternalFunctionExpression, + Expr_ifExpression, +) +from pyomo.core.expr.logical_expr import BooleanExpression +from pyomo.core.expr.relational_expr import ( + EqualityExpression, + InequalityExpression, + RangedExpression, ) from pyomo.core.expr.numvalue import native_numeric_types, native_types, value from pyomo.core.expr.visitor import StreamBasedExpressionVisitor from pyomo.repn.util import BeforeChildDispatcher, ExitNodeDispatcher inf = float('inf') +logger = logging.getLogger(__name__) class ExpressionBoundsBeforeChildDispatcher(BeforeChildDispatcher): @@ -226,20 +236,20 @@ def _handle_named_expression(visitor, node, arg): } -_operator_dispatcher = ExitNodeDispatcher( - { - ProductExpression: _handle_ProductExpression, - DivisionExpression: _handle_DivisionExpression, - PowExpression: _handle_PowExpression, - AbsExpression: _handle_AbsExpression, - SumExpression: _handle_SumExpression, - MonomialTermExpression: _handle_ProductExpression, - NegationExpression: _handle_NegationExpression, - UnaryFunctionExpression: _handle_UnaryFunctionExpression, - LinearExpression: _handle_SumExpression, - Expression: _handle_named_expression, - } -) +class ExpressionBoundsExitNodeDispatcher(ExitNodeDispatcher): + def unexpected_expression_type(self, visitor, node, *args): + if isinstance(node, NumericExpression): + ans = -inf, inf + elif isinstance(node, BooleanExpression): + ans = Bool(False), Bool(True) + else: + super().unexpected_expression_type(visitor, node, *args) + logger.warning( + f"Unexpected expression node type '{type(node).__name__}' " + f"found while walking expression tree; returning {ans} " + "for the expression bounds." + ) + return ans class ExpressionBoundsVisitor(StreamBasedExpressionVisitor): @@ -264,6 +274,27 @@ class ExpressionBoundsVisitor(StreamBasedExpressionVisitor): the computed bounds should be valid. """ + _before_child_handlers = ExpressionBoundsBeforeChildDispatcher() + _operator_dispatcher = ExpressionBoundsExitNodeDispatcher( + { + ProductExpression: _handle_ProductExpression, + DivisionExpression: _handle_DivisionExpression, + PowExpression: _handle_PowExpression, + AbsExpression: _handle_AbsExpression, + SumExpression: _handle_SumExpression, + MonomialTermExpression: _handle_ProductExpression, + NegationExpression: _handle_NegationExpression, + UnaryFunctionExpression: _handle_UnaryFunctionExpression, + LinearExpression: _handle_SumExpression, + Expression: _handle_named_expression, + ExternalFunctionExpression: _handle_unknowable_bounds, + EqualityExpression: _handle_equality, + InequalityExpression: _handle_inequality, + RangedExpression: _handle_ranged, + Expr_ifExpression: _handle_expr_if, + } + ) + def __init__( self, leaf_bounds=None, diff --git a/pyomo/contrib/fbbt/tests/test_expression_bounds_walker.py b/pyomo/contrib/fbbt/tests/test_expression_bounds_walker.py index 612a3101ef7..8b30ffdef4b 100644 --- a/pyomo/contrib/fbbt/tests/test_expression_bounds_walker.py +++ b/pyomo/contrib/fbbt/tests/test_expression_bounds_walker.py @@ -10,10 +10,33 @@ # ___________________________________________________________________________ import math -from pyomo.environ import exp, log, log10, sin, cos, tan, asin, acos, atan, sqrt import pyomo.common.unittest as unittest -from pyomo.contrib.fbbt.expression_bounds_walker import ExpressionBoundsVisitor -from pyomo.core import Any, ConcreteModel, Expression, Param, Var + +from pyomo.environ import ( + exp, + log, + log10, + sin, + cos, + tan, + asin, + acos, + atan, + sqrt, + inequality, + Expr_if, + Any, + ConcreteModel, + Expression, + Param, + Var, +) + +from pyomo.common.errors import DeveloperError +from pyomo.common.log import LoggingIntercept +from pyomo.contrib.fbbt.expression_bounds_walker import ExpressionBoundsVisitor, inf +from pyomo.contrib.fbbt.interval import _true, _false +from pyomo.core.expr import ExpressionBase, NumericExpression, BooleanExpression class TestExpressionBoundsWalker(unittest.TestCase): From 64d4f473d566b22f27369a1c1b1c5aaa01198467 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 8 Feb 2024 15:40:22 -0700 Subject: [PATCH 089/103] Add support for walking logical expressions --- .../contrib/fbbt/expression_bounds_walker.py | 25 ++++++ pyomo/contrib/fbbt/interval.py | 89 +++++++++++++++++++ .../tests/test_expression_bounds_walker.py | 79 ++++++++++++++++ 3 files changed, 193 insertions(+) diff --git a/pyomo/contrib/fbbt/expression_bounds_walker.py b/pyomo/contrib/fbbt/expression_bounds_walker.py index 31e52e0dc79..a32d138c52b 100644 --- a/pyomo/contrib/fbbt/expression_bounds_walker.py +++ b/pyomo/contrib/fbbt/expression_bounds_walker.py @@ -13,6 +13,11 @@ from math import pi from pyomo.common.collections import ComponentMap from pyomo.contrib.fbbt.interval import ( + Bool, + eq, + ineq, + ranged, + if_, add, acos, asin, @@ -222,6 +227,26 @@ def _handle_named_expression(visitor, node, arg): return arg +def _handle_unknowable_bounds(visitor, node, arg): + return -inf, inf + + +def _handle_equality(visitor, node, arg1, arg2): + return eq(*arg1, *arg2) + + +def _handle_inequality(visitor, node, arg1, arg2): + return ineq(*arg1, *arg2) + + +def _handle_ranged(visitor, node, arg1, arg2, arg3): + return ranged(*arg1, *arg2, *arg3) + + +def _handle_expr_if(visitor, node, arg1, arg2, arg3): + return if_(*arg1, *arg2, *arg3) + + _unary_function_dispatcher = { 'exp': _handle_exp, 'log': _handle_log, diff --git a/pyomo/contrib/fbbt/interval.py b/pyomo/contrib/fbbt/interval.py index fd86af4c106..53c236850d9 100644 --- a/pyomo/contrib/fbbt/interval.py +++ b/pyomo/contrib/fbbt/interval.py @@ -17,6 +17,95 @@ inf = float('inf') +class bool_(object): + def __init__(self, val): + self._val = val + + def __bool__(self): + return self._val + + def _op(self, *others): + raise ValueError( + f"{self._val!r} ({type(self._val).__name__}) is not a valid numeric type. " + f"Cannot compute bounds on expression." + ) + + def __repr__(self): + return repr(self._val) + + __float__ = _op + __int__ = _op + __abs__ = _op + __neg__ = _op + __add__ = _op + __sub__ = _op + __mul__ = _op + __div__ = _op + __pow__ = _op + __radd__ = _op + __rsub__ = _op + __rmul__ = _op + __rdiv__ = _op + __rpow__ = _op + + +_true = bool_(True) +_false = bool_(False) + + +def Bool(val): + return _true if val else _false + + +def ineq(xl, xu, yl, yu): + ans = [] + if yl < xu: + ans.append(_false) + if xl <= yu: + ans.append(_true) + assert ans + if len(ans) == 1: + ans.append(ans[0]) + return tuple(ans) + + +def eq(xl, xu, yl, yu): + ans = [] + if xl != xu or yl != yu or xl != yl: + ans.append(_false) + if xl <= yu and yl <= xu: + ans.append(_true) + assert ans + if len(ans) == 1: + ans.append(ans[0]) + return tuple(ans) + + +def ranged(xl, xu, yl, yu, zl, zu): + lb = ineq(xl, xu, yl, yu) + ub = ineq(yl, yu, zl, zu) + ans = [] + if not lb[0] or not ub[0]: + ans.append(_false) + if lb[1] and ub[1]: + ans.append(_true) + if len(ans) == 1: + ans.append(ans[0]) + return tuple(ans) + + +def if_(il, iu, tl, tu, fl, fu): + l = [] + u = [] + if iu: + l.append(tl) + u.append(tu) + if not il: + l.append(fl) + u.append(fu) + return min(l), max(u) + + def add(xl, xu, yl, yu): return xl + yl, xu + yu diff --git a/pyomo/contrib/fbbt/tests/test_expression_bounds_walker.py b/pyomo/contrib/fbbt/tests/test_expression_bounds_walker.py index 8b30ffdef4b..75d273422d1 100644 --- a/pyomo/contrib/fbbt/tests/test_expression_bounds_walker.py +++ b/pyomo/contrib/fbbt/tests/test_expression_bounds_walker.py @@ -334,3 +334,82 @@ def test_invalid_complex(self): r"complex numbers. Encountered when processing \(4\+5j\)", ): lb, ub = visitor.walk_expression(m.p + m.y) + + def test_inequality(self): + m = self.make_model() + visitor = ExpressionBoundsVisitor() + self.assertEqual(visitor.walk_expression(m.z <= m.y), (_true, _true)) + self.assertEqual(visitor.walk_expression(m.y <= m.z), (_false, _false)) + self.assertEqual(visitor.walk_expression(m.y <= m.x), (_false, _true)) + + def test_equality(self): + m = self.make_model() + m.p = Param(initialize=5) + visitor = ExpressionBoundsVisitor() + self.assertEqual(visitor.walk_expression(m.y == m.z), (_false, _false)) + self.assertEqual(visitor.walk_expression(m.y == m.x), (_false, _true)) + self.assertEqual(visitor.walk_expression(m.p == m.p), (_true, _true)) + + def test_ranged(self): + m = self.make_model() + visitor = ExpressionBoundsVisitor() + self.assertEqual( + visitor.walk_expression(inequality(m.z, m.y, 5)), (_true, _true) + ) + self.assertEqual( + visitor.walk_expression(inequality(m.y, m.z, m.y)), (_false, _false) + ) + self.assertEqual( + visitor.walk_expression(inequality(m.y, m.x, m.y)), (_false, _true) + ) + + def test_expr_if(self): + m = self.make_model() + visitor = ExpressionBoundsVisitor() + self.assertEqual( + visitor.walk_expression(Expr_if(IF=m.z <= m.y, THEN=m.z, ELSE=m.y)), + m.z.bounds, + ) + self.assertEqual( + visitor.walk_expression(Expr_if(IF=m.z >= m.y, THEN=m.z, ELSE=m.y)), + m.y.bounds, + ) + self.assertEqual( + visitor.walk_expression(Expr_if(IF=m.y <= m.x, THEN=m.y, ELSE=m.x)), (-2, 5) + ) + + def test_unknown_classes(self): + class UnknownNumeric(NumericExpression): + pass + + class UnknownLogic(BooleanExpression): + def nargs(self): + return 0 + + class UnknownOther(ExpressionBase): + @property + def args(self): + return () + + def nargs(self): + return 0 + + visitor = ExpressionBoundsVisitor() + with LoggingIntercept() as LOG: + self.assertEqual(visitor.walk_expression(UnknownNumeric(())), (-inf, inf)) + self.assertEqual( + LOG.getvalue(), + "Unexpected expression node type 'UnknownNumeric' found while walking " + "expression tree; returning (-inf, inf) for the expression bounds.\n", + ) + with LoggingIntercept() as LOG: + self.assertEqual(visitor.walk_expression(UnknownLogic(())), (_false, _true)) + self.assertEqual( + LOG.getvalue(), + "Unexpected expression node type 'UnknownLogic' found while walking " + "expression tree; returning (False, True) for the expression bounds.\n", + ) + with self.assertRaisesRegex( + DeveloperError, "Unexpected expression node type 'UnknownOther' found" + ): + visitor.walk_expression(UnknownOther()) From 45ca395d213dfa74393cd52c094e73992bccfac3 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 8 Feb 2024 17:17:51 -0700 Subject: [PATCH 090/103] Updating tests to reflect changes in the before child dispatcher --- pyomo/repn/tests/test_util.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyomo/repn/tests/test_util.py b/pyomo/repn/tests/test_util.py index 8ea6bda83b9..48d78c60d6e 100644 --- a/pyomo/repn/tests/test_util.py +++ b/pyomo/repn/tests/test_util.py @@ -717,14 +717,14 @@ class UnknownExpression(NumericExpression): DeveloperError, r".*Unexpected expression node type 'UnknownExpression'" ): end[node.__class__](None, node, *node.args) - self.assertEqual(len(end), 8) + self.assertEqual(len(end), 9) node = UnknownExpression((6, 7)) with self.assertRaisesRegex( DeveloperError, r".*Unexpected expression node type 'UnknownExpression'" ): end[node.__class__, 6, 7](None, node, *node.args) - self.assertEqual(len(end), 8) + self.assertEqual(len(end), 10) def test_BeforeChildDispatcher_registration(self): class BeforeChildDispatcherTester(BeforeChildDispatcher): @@ -758,7 +758,7 @@ def evaluate(self, node): self.assertEqual(ans, (False, (_CONSTANT, InvalidNumber(node)))) self.assertEqual( ''.join(ans[1][1].causes), - "'string' () is not a valid numeric type", + "'string' (str) is not a valid numeric type", ) self.assertIs(bcd[str], bcd._before_string) self.assertEqual(len(bcd), 2) @@ -768,9 +768,9 @@ def evaluate(self, node): self.assertEqual(ans, (False, (_CONSTANT, InvalidNumber(node)))) self.assertEqual( ''.join(ans[1][1].causes), - "True () is not a valid numeric type", + "True (bool) is not a valid numeric type", ) - self.assertIs(bcd[bool], bcd._before_invalid) + self.assertIs(bcd[bool], bcd._before_native_logical) self.assertEqual(len(bcd), 3) node = 1j @@ -794,7 +794,7 @@ class new_int(int): ans = bcd[node.__class__](None, node) self.assertEqual(ans, (False, (_CONSTANT, InvalidNumber([])))) self.assertEqual( - ''.join(ans[1][1].causes), "[] () is not a valid numeric type" + ''.join(ans[1][1].causes), "[] (list) is not a valid numeric type" ) self.assertIs(bcd[list], bcd._before_invalid) self.assertEqual(len(bcd), 6) From 67acc27477914b7e254fa4df5c978281448f50ee Mon Sep 17 00:00:00 2001 From: John Siirola Date: Fri, 9 Feb 2024 09:05:48 -0700 Subject: [PATCH 091/103] NFC: apply black --- pyomo/repn/tests/test_util.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pyomo/repn/tests/test_util.py b/pyomo/repn/tests/test_util.py index 48d78c60d6e..cce10e58334 100644 --- a/pyomo/repn/tests/test_util.py +++ b/pyomo/repn/tests/test_util.py @@ -757,8 +757,7 @@ def evaluate(self, node): ans = bcd[node.__class__](None, node) self.assertEqual(ans, (False, (_CONSTANT, InvalidNumber(node)))) self.assertEqual( - ''.join(ans[1][1].causes), - "'string' (str) is not a valid numeric type", + ''.join(ans[1][1].causes), "'string' (str) is not a valid numeric type" ) self.assertIs(bcd[str], bcd._before_string) self.assertEqual(len(bcd), 2) @@ -767,8 +766,7 @@ def evaluate(self, node): ans = bcd[node.__class__](None, node) self.assertEqual(ans, (False, (_CONSTANT, InvalidNumber(node)))) self.assertEqual( - ''.join(ans[1][1].causes), - "True (bool) is not a valid numeric type", + ''.join(ans[1][1].causes), "True (bool) is not a valid numeric type" ) self.assertIs(bcd[bool], bcd._before_native_logical) self.assertEqual(len(bcd), 3) From 227836df546cef9729f6af5dbb32664e75f3f3ac Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Fri, 9 Feb 2024 13:11:31 -0700 Subject: [PATCH 092/103] remove unused imports --- pyomo/contrib/incidence_analysis/config.py | 2 +- pyomo/contrib/incidence_analysis/incidence.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/config.py b/pyomo/contrib/incidence_analysis/config.py index d055be478fe..72d1a41ac74 100644 --- a/pyomo/contrib/incidence_analysis/config.py +++ b/pyomo/contrib/incidence_analysis/config.py @@ -14,7 +14,7 @@ import enum from pyomo.common.config import ConfigDict, ConfigValue, InEnum from pyomo.common.modeling import NOTSET -from pyomo.repn.plugins.nl_writer import AMPLRepnVisitor, AMPLRepn, text_nl_template +from pyomo.repn.plugins.nl_writer import AMPLRepnVisitor, text_nl_template from pyomo.repn.util import FileDeterminism, FileDeterminism_to_SortComponents diff --git a/pyomo/contrib/incidence_analysis/incidence.py b/pyomo/contrib/incidence_analysis/incidence.py index 636a400def4..13e9997d6c3 100644 --- a/pyomo/contrib/incidence_analysis/incidence.py +++ b/pyomo/contrib/incidence_analysis/incidence.py @@ -16,9 +16,8 @@ from pyomo.core.expr.visitor import identify_variables from pyomo.core.expr.numvalue import value as pyo_value from pyomo.repn import generate_standard_repn -from pyomo.repn.plugins.nl_writer import AMPLRepnVisitor, AMPLRepn, text_nl_template -from pyomo.repn.util import FileDeterminism, FileDeterminism_to_SortComponents from pyomo.util.subsystems import TemporarySubsystemManager +from pyomo.repn.plugins.nl_writer import AMPLRepn from pyomo.contrib.incidence_analysis.config import ( IncidenceMethod, get_config_from_kwds, From 677ebc9e67d363d364fa6f771fa610fe2bda2bbc Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Fri, 9 Feb 2024 13:14:24 -0700 Subject: [PATCH 093/103] remove unused imports --- pyomo/contrib/incidence_analysis/interface.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyomo/contrib/incidence_analysis/interface.py b/pyomo/contrib/incidence_analysis/interface.py index 41f0ece3a75..b6e6583da88 100644 --- a/pyomo/contrib/incidence_analysis/interface.py +++ b/pyomo/contrib/incidence_analysis/interface.py @@ -45,8 +45,6 @@ ) from pyomo.contrib.incidence_analysis.incidence import get_incident_variables from pyomo.contrib.pynumero.asl import AmplInterface -from pyomo.repn.plugins.nl_writer import AMPLRepnVisitor, AMPLRepn, text_nl_template -from pyomo.repn.util import FileDeterminism, FileDeterminism_to_SortComponents pyomo_nlp, pyomo_nlp_available = attempt_import( 'pyomo.contrib.pynumero.interfaces.pyomo_nlp' From b90fb0c1b18ac14d15121c857f344ac1875f32a7 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 12 Feb 2024 17:40:35 -0700 Subject: [PATCH 094/103] NFC: add docstrings, reformat long lines --- pyomo/contrib/fbbt/interval.py | 197 +++++++++++++++++++++------------ 1 file changed, 129 insertions(+), 68 deletions(-) diff --git a/pyomo/contrib/fbbt/interval.py b/pyomo/contrib/fbbt/interval.py index 53c236850d9..339d547b7d4 100644 --- a/pyomo/contrib/fbbt/interval.py +++ b/pyomo/contrib/fbbt/interval.py @@ -58,6 +58,14 @@ def Bool(val): def ineq(xl, xu, yl, yu): + """Compute the "bounds" on an InequalityExpression + + Note this is *not* performing interval arithmetic: we are + calculating the "bounds" on a RelationalExpression (whose domain is + {True, False}). Therefore we are determining if `x` can be less + than `y`, `x` can not be less than `y`, or both. + + """ ans = [] if yl < xu: ans.append(_false) @@ -70,6 +78,14 @@ def ineq(xl, xu, yl, yu): def eq(xl, xu, yl, yu): + """Compute the "bounds" on an EqualityExpression + + Note this is *not* performing interval arithmetic: we are + calculating the "bounds" on a RelationalExpression (whose domain is + {True, False}). Therefore we are determining if `x` can be equal to + `y`, `x` can not be equal to `y`, or both. + + """ ans = [] if xl != xu or yl != yu or xl != yl: ans.append(_false) @@ -82,6 +98,14 @@ def eq(xl, xu, yl, yu): def ranged(xl, xu, yl, yu, zl, zu): + """Compute the "bounds" on a RangedExpression + + Note this is *not* performing interval arithmetic: we are + calculating the "bounds" on a RelationalExpression (whose domain is + {True, False}). Therefore we are determining if `y` can be between + `z` and `z`, `y` can be outside the range `x` and `z`, or both. + + """ lb = ineq(xl, xu, yl, yu) ub = ineq(yl, yu, zl, zu) ans = [] @@ -128,12 +152,18 @@ def mul(xl, xu, yl, yu): def inv(xl, xu, feasibility_tol): - """ - The case where xl is very slightly positive but should be very slightly negative (or xu is very slightly negative - but should be very slightly positive) should not be an issue. Suppose xu is 2 and xl is 1e-15 but should be -1e-15. - The bounds obtained from this function will be [0.5, 1e15] or [0.5, inf), depending on the value of - feasibility_tol. The true bounds are (-inf, -1e15] U [0.5, inf), where U is union. The exclusion of (-inf, -1e15] - should be acceptable. Additionally, it very important to return a non-negative interval when xl is non-negative. + """Compute the inverse of an interval + + The case where xl is very slightly positive but should be very + slightly negative (or xu is very slightly negative but should be + very slightly positive) should not be an issue. Suppose xu is 2 and + xl is 1e-15 but should be -1e-15. The bounds obtained from this + function will be [0.5, 1e15] or [0.5, inf), depending on the value + of feasibility_tol. The true bounds are (-inf, -1e15] U [0.5, inf), + where U is union. The exclusion of (-inf, -1e15] should be + acceptable. Additionally, it very important to return a non-negative + interval when xl is non-negative. + """ if xu - xl <= -feasibility_tol: raise InfeasibleConstraintException( @@ -178,9 +208,8 @@ def power(xl, xu, yl, yu, feasibility_tol): Compute bounds on x**y. """ if xl > 0: - """ - If x is always positive, things are simple. We only need to worry about the sign of y. - """ + # If x is always positive, things are simple. We only need to + # worry about the sign of y. if yl < 0 < yu: lb = min(xu**yl, xl**yu) ub = max(xl**yl, xu**yu) @@ -270,14 +299,15 @@ def power(xl, xu, yl, yu, feasibility_tol): def _inverse_power1(zl, zu, yl, yu, orig_xl, orig_xu, feasibility_tol): - """ - z = x**y => compute bounds on x. + """z = x**y => compute bounds on x. First, start by computing bounds on x with x = exp(ln(z) / y) - However, if y is an integer, then x can be negative, so there are several special cases. See the docs below. + However, if y is an integer, then x can be negative, so there are + several special cases. See the docs below. + """ xl, xu = log(zl, zu) xl, xu = div(xl, xu, yl, yu, feasibility_tol) @@ -288,22 +318,31 @@ def _inverse_power1(zl, zu, yl, yu, orig_xl, orig_xu, feasibility_tol): y = yl if y == 0: # Anything to the power of 0 is 1, so if y is 0, then x can be anything - # (assuming zl <= 1 <= zu, which is enforced when traversing the tree in the other direction) + # (assuming zl <= 1 <= zu, which is enforced when traversing + # the tree in the other direction) xl = -inf xu = inf elif y % 2 == 0: - """ - if y is even, then there are two primary cases (note that it is much easier to walk through these - while looking at plots): + """if y is even, then there are two primary cases (note that it is much + easier to walk through these while looking at plots): + case 1: y is positive - x**y is convex, positive, and symmetric. The bounds on x depend on the lower bound of z. If zl <= 0, - then xl should simply be -xu. However, if zl > 0, then we may be able to say something better. For - example, if the original lower bound on x is positive, then we can keep xl computed from - x = exp(ln(z) / y). Furthermore, if the original lower bound on x is larger than -xl computed from - x = exp(ln(z) / y), then we can still keep the xl computed from x = exp(ln(z) / y). Similar logic - applies to the upper bound of x. + + x**y is convex, positive, and symmetric. The bounds on x + depend on the lower bound of z. If zl <= 0, then xl + should simply be -xu. However, if zl > 0, then we may be + able to say something better. For example, if the + original lower bound on x is positive, then we can keep + xl computed from x = exp(ln(z) / y). Furthermore, if the + original lower bound on x is larger than -xl computed + from x = exp(ln(z) / y), then we can still keep the xl + computed from x = exp(ln(z) / y). Similar logic applies + to the upper bound of x. + case 2: y is negative + The ideas are similar to case 1. + """ if zu + feasibility_tol < 0: raise InfeasibleConstraintException( @@ -351,16 +390,25 @@ def _inverse_power1(zl, zu, yl, yu, orig_xl, orig_xu, feasibility_tol): xl = _xl xu = _xu else: # y % 2 == 1 - """ - y is odd. + """y is odd. + Case 1: y is positive - x**y is monotonically increasing. If y is positive, then we can can compute the bounds on x using - x = z**(1/y) and the signs on xl and xu depend on the signs of zl and zu. + + x**y is monotonically increasing. If y is positive, then + we can can compute the bounds on x using x = z**(1/y) + and the signs on xl and xu depend on the signs of zl and + zu. + Case 2: y is negative - Again, this is easier to visualize with a plot. x**y approaches zero when x approaches -inf or inf. - Thus, if zl < 0 < zu, then no bounds can be inferred for x. If z is positive (zl >=0 ) then we can - use the bounds computed from x = exp(ln(z) / y). If z is negative (zu <= 0), then we live in the - bottom left quadrant, xl depends on zu, and xu depends on zl. + + Again, this is easier to visualize with a plot. x**y + approaches zero when x approaches -inf or inf. Thus, if + zl < 0 < zu, then no bounds can be inferred for x. If z + is positive (zl >=0 ) then we can use the bounds + computed from x = exp(ln(z) / y). If z is negative (zu + <= 0), then we live in the bottom left quadrant, xl + depends on zu, and xu depends on zl. + """ if y > 0: xl = abs(zl) ** (1.0 / y) @@ -387,12 +435,13 @@ def _inverse_power1(zl, zu, yl, yu, orig_xl, orig_xu, feasibility_tol): def _inverse_power2(zl, zu, xl, xu, feasiblity_tol): - """ - z = x**y => compute bounds on y + """z = x**y => compute bounds on y y = ln(z) / ln(x) - This function assumes the exponent can be fractional, so x must be positive. This method should not be called - if the exponent is an integer. + This function assumes the exponent can be fractional, so x must be + positive. This method should not be called if the exponent is an + integer. + """ if xu <= 0: raise IntervalException( @@ -480,10 +529,12 @@ def sin(xl, xu): ub: float """ - # if there is a minimum between xl and xu, then the lower bound is -1. Minimums occur at 2*pi*n - pi/2 - # find the minimum value of i such that 2*pi*i - pi/2 >= xl. Then round i up. If 2*pi*i - pi/2 is still less - # than or equal to xu, then there is a minimum between xl and xu. Thus the lb is -1. Otherwise, the minimum - # occurs at either xl or xu + # if there is a minimum between xl and xu, then the lower bound is + # -1. Minimums occur at 2*pi*n - pi/2 find the minimum value of i + # such that 2*pi*i - pi/2 >= xl. Then round i up. If 2*pi*i - pi/2 + # is still less than or equal to xu, then there is a minimum between + # xl and xu. Thus the lb is -1. Otherwise, the minimum occurs at + # either xl or xu if xl <= -inf or xu >= inf: return -1, 1 pi = math.pi @@ -495,7 +546,8 @@ def sin(xl, xu): else: lb = min(math.sin(xl), math.sin(xu)) - # if there is a maximum between xl and xu, then the upper bound is 1. Maximums occur at 2*pi*n + pi/2 + # if there is a maximum between xl and xu, then the upper bound is + # 1. Maximums occur at 2*pi*n + pi/2 i = (xu - pi / 2) / (2 * pi) i = math.floor(i) x_at_max = 2 * pi * i + pi / 2 @@ -521,10 +573,12 @@ def cos(xl, xu): ub: float """ - # if there is a minimum between xl and xu, then the lower bound is -1. Minimums occur at 2*pi*n - pi - # find the minimum value of i such that 2*pi*i - pi >= xl. Then round i up. If 2*pi*i - pi/2 is still less - # than or equal to xu, then there is a minimum between xl and xu. Thus the lb is -1. Otherwise, the minimum - # occurs at either xl or xu + # if there is a minimum between xl and xu, then the lower bound is + # -1. Minimums occur at 2*pi*n - pi find the minimum value of i such + # that 2*pi*i - pi >= xl. Then round i up. If 2*pi*i - pi/2 is still + # less than or equal to xu, then there is a minimum between xl and + # xu. Thus the lb is -1. Otherwise, the minimum occurs at either xl + # or xu if xl <= -inf or xu >= inf: return -1, 1 pi = math.pi @@ -536,7 +590,8 @@ def cos(xl, xu): else: lb = min(math.cos(xl), math.cos(xu)) - # if there is a maximum between xl and xu, then the upper bound is 1. Maximums occur at 2*pi*n + # if there is a maximum between xl and xu, then the upper bound is + # 1. Maximums occur at 2*pi*n i = (xu) / (2 * pi) i = math.floor(i) x_at_max = 2 * pi * i @@ -562,10 +617,12 @@ def tan(xl, xu): ub: float """ - # tan goes to -inf and inf at every pi*i + pi/2 (integer i). If one of these values is between xl and xu, then - # the lb is -inf and the ub is inf. Otherwise the minimum occurs at xl and the maximum occurs at xu. - # find the minimum value of i such that pi*i + pi/2 >= xl. Then round i up. If pi*i + pi/2 is still less - # than or equal to xu, then there is an undefined point between xl and xu. + # tan goes to -inf and inf at every pi*i + pi/2 (integer i). If one + # of these values is between xl and xu, then the lb is -inf and the + # ub is inf. Otherwise the minimum occurs at xl and the maximum + # occurs at xu. find the minimum value of i such that pi*i + pi/2 + # >= xl. Then round i up. If pi*i + pi/2 is still less than or equal + # to xu, then there is an undefined point between xl and xu. if xl <= -inf or xu >= inf: return -inf, inf pi = math.pi @@ -609,12 +666,12 @@ def asin(xl, xu, yl, yu, feasibility_tol): if yl <= -inf: lb = yl elif xl <= math.sin(yl) <= xu: - # if sin(yl) >= xl then yl satisfies the bounds on x, and the lower bound of y cannot be improved + # if sin(yl) >= xl then yl satisfies the bounds on x, and the + # lower bound of y cannot be improved lb = yl elif math.sin(yl) < xl: - """ - we can only push yl up from its current value to the next lowest value such that xl = sin(y). In other words, - we need to + """we can only push yl up from its current value to the next lowest + value such that xl = sin(y). In other words, we need to min y s.t. @@ -622,19 +679,21 @@ def asin(xl, xu, yl, yu, feasibility_tol): y >= yl globally. + """ - # first find the next minimum of x = sin(y). Minimums occur at y = 2*pi*n - pi/2 for integer n. + # first find the next minimum of x = sin(y). Minimums occur at y + # = 2*pi*n - pi/2 for integer n. i = (yl + pi / 2) / (2 * pi) i1 = math.floor(i) i2 = math.ceil(i) i1 = 2 * pi * i1 - pi / 2 i2 = 2 * pi * i2 - pi / 2 - # now find the next value of y such that xl = sin(y). This can be computed by a distance from the minimum (i). + # now find the next value of y such that xl = sin(y). This can + # be computed by a distance from the minimum (i). y_tmp = math.asin(xl) # this will give me a value between -pi/2 and pi/2 - dist = y_tmp - ( - -pi / 2 - ) # this is the distance between the minimum of the sin function and a value that - # satisfies xl = sin(y) + dist = y_tmp - (-pi / 2) + # this is the distance between the minimum of the sin function + # and a value that satisfies xl = sin(y) lb1 = i1 + dist lb2 = i2 + dist if lb1 >= yl - feasibility_tol: @@ -722,12 +781,12 @@ def acos(xl, xu, yl, yu, feasibility_tol): if yl <= -inf: lb = yl elif xl <= math.cos(yl) <= xu: - # if xl <= cos(yl) <= xu then yl satisfies the bounds on x, and the lower bound of y cannot be improved + # if xl <= cos(yl) <= xu then yl satisfies the bounds on x, and + # the lower bound of y cannot be improved lb = yl elif math.cos(yl) < xl: - """ - we can only push yl up from its current value to the next lowest value such that xl = cos(y). In other words, - we need to + """we can only push yl up from its current value to the next lowest + value such that xl = cos(y). In other words, we need to min y s.t. @@ -735,19 +794,21 @@ def acos(xl, xu, yl, yu, feasibility_tol): y >= yl globally. + """ - # first find the next minimum of x = cos(y). Minimums occur at y = 2*pi*n - pi for integer n. + # first find the next minimum of x = cos(y). Minimums occur at y + # = 2*pi*n - pi for integer n. i = (yl + pi) / (2 * pi) i1 = math.floor(i) i2 = math.ceil(i) i1 = 2 * pi * i1 - pi i2 = 2 * pi * i2 - pi - # now find the next value of y such that xl = cos(y). This can be computed by a distance from the minimum (i). + # now find the next value of y such that xl = cos(y). This can + # be computed by a distance from the minimum (i). y_tmp = math.acos(xl) # this will give me a value between 0 and pi - dist = ( - pi - y_tmp - ) # this is the distance between the minimum of the sin function and a value that - # satisfies xl = sin(y) + dist = pi - y_tmp + # this is the distance between the minimum of the sin function + # and a value that satisfies xl = sin(y) lb1 = i1 + dist lb2 = i2 + dist if lb1 >= yl - feasibility_tol: From 3d14132f02340a928c63a78c2c23d2358968f140 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 12 Feb 2024 17:42:41 -0700 Subject: [PATCH 095/103] Make bool_ private; rename Bool -> BoolFlag so usage doesn't look like a bool --- pyomo/contrib/fbbt/expression_bounds_walker.py | 6 +++--- pyomo/contrib/fbbt/interval.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/fbbt/expression_bounds_walker.py b/pyomo/contrib/fbbt/expression_bounds_walker.py index a32d138c52b..340af94c83e 100644 --- a/pyomo/contrib/fbbt/expression_bounds_walker.py +++ b/pyomo/contrib/fbbt/expression_bounds_walker.py @@ -13,7 +13,7 @@ from math import pi from pyomo.common.collections import ComponentMap from pyomo.contrib.fbbt.interval import ( - Bool, + BoolFlag, eq, ineq, ranged, @@ -81,7 +81,7 @@ def _before_native_numeric(visitor, child): @staticmethod def _before_native_logical(visitor, child): - return False, (Bool(child), Bool(child)) + return False, (BoolFlag(child), BoolFlag(child)) @staticmethod def _before_var(visitor, child): @@ -266,7 +266,7 @@ def unexpected_expression_type(self, visitor, node, *args): if isinstance(node, NumericExpression): ans = -inf, inf elif isinstance(node, BooleanExpression): - ans = Bool(False), Bool(True) + ans = BoolFlag(False), BoolFlag(True) else: super().unexpected_expression_type(visitor, node, *args) logger.warning( diff --git a/pyomo/contrib/fbbt/interval.py b/pyomo/contrib/fbbt/interval.py index 339d547b7d4..8bebe128988 100644 --- a/pyomo/contrib/fbbt/interval.py +++ b/pyomo/contrib/fbbt/interval.py @@ -17,7 +17,7 @@ inf = float('inf') -class bool_(object): +class _bool_flag(object): def __init__(self, val): self._val = val @@ -49,11 +49,11 @@ def __repr__(self): __rpow__ = _op -_true = bool_(True) -_false = bool_(False) +_true = _bool_flag(True) +_false = _bool_flag(False) -def Bool(val): +def BoolFlag(val): return _true if val else _false From e8ba13e43dc16ca829b8ecda15f1dc7ce8635b19 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 13 Feb 2024 09:12:24 -0700 Subject: [PATCH 096/103] Minor update to tests --- pyomo/repn/tests/test_util.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyomo/repn/tests/test_util.py b/pyomo/repn/tests/test_util.py index cce10e58334..ac3f7e62791 100644 --- a/pyomo/repn/tests/test_util.py +++ b/pyomo/repn/tests/test_util.py @@ -699,6 +699,7 @@ def test_ExitNodeDispatcher_registration(self): self.assertEqual(end[node.__class__, 3, 4, 5, 6](None, node, *node.args), 6) self.assertEqual(len(end), 7) + # We don't cache etypes with more than 3 arguments self.assertNotIn((SumExpression, 3, 4, 5, 6), end) class NewProductExpression(ProductExpression): @@ -718,6 +719,7 @@ class UnknownExpression(NumericExpression): ): end[node.__class__](None, node, *node.args) self.assertEqual(len(end), 9) + self.assertIn(UnknownExpression, end) node = UnknownExpression((6, 7)) with self.assertRaisesRegex( @@ -725,6 +727,8 @@ class UnknownExpression(NumericExpression): ): end[node.__class__, 6, 7](None, node, *node.args) self.assertEqual(len(end), 10) + self.assertIn((UnknownExpression, 6, 7), end) + def test_BeforeChildDispatcher_registration(self): class BeforeChildDispatcherTester(BeforeChildDispatcher): From 94d131fca8aba4ec45271f51dcd12722025e97a5 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 13 Feb 2024 09:16:15 -0700 Subject: [PATCH 097/103] NFC: apply black --- pyomo/repn/tests/test_util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/repn/tests/test_util.py b/pyomo/repn/tests/test_util.py index ac3f7e62791..c4902a7064d 100644 --- a/pyomo/repn/tests/test_util.py +++ b/pyomo/repn/tests/test_util.py @@ -729,7 +729,6 @@ class UnknownExpression(NumericExpression): self.assertEqual(len(end), 10) self.assertIn((UnknownExpression, 6, 7), end) - def test_BeforeChildDispatcher_registration(self): class BeforeChildDispatcherTester(BeforeChildDispatcher): @staticmethod From a77a19b5302544aad90b457307f1a5f650583402 Mon Sep 17 00:00:00 2001 From: Zedong Date: Tue, 13 Feb 2024 13:24:13 -0500 Subject: [PATCH 098/103] Update pyomo/contrib/mindtpy/util.py Co-authored-by: Bethany Nicholson --- pyomo/contrib/mindtpy/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 6061da0f0d9..575544eed5c 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -1024,6 +1024,6 @@ def set_var_valid_value( var.set_value(0) else: raise ValueError( - "copy_var_list_values failed with variable {}, value = {} and rounded value = {}" + "set_var_valid_value failed with variable {}, value = {} and rounded value = {}" "".format(var.name, var_val, rounded_val) ) From a0997d824d9079cc3621b4cb77e2d5933f16804c Mon Sep 17 00:00:00 2001 From: Zedong Date: Tue, 13 Feb 2024 13:24:28 -0500 Subject: [PATCH 099/103] Update pyomo/contrib/mindtpy/util.py Co-authored-by: Bethany Nicholson --- pyomo/contrib/mindtpy/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index 575544eed5c..fa6aec7f08f 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -944,7 +944,7 @@ def copy_var_list_values( Sets to zero for NonNegativeReals if necessary from_list : list - The variables that provides the values to copy from. + The variables that provide the values to copy from. to_list : list The variables that need to set value. config : ConfigBlock From 6c9fa3ee64bbaeb9d8bce565a857b584b886a401 Mon Sep 17 00:00:00 2001 From: ZedongPeng Date: Tue, 13 Feb 2024 13:34:35 -0500 Subject: [PATCH 100/103] update the differentiate.Modes --- pyomo/contrib/mindtpy/util.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyomo/contrib/mindtpy/util.py b/pyomo/contrib/mindtpy/util.py index fa6aec7f08f..69c7ca5030a 100644 --- a/pyomo/contrib/mindtpy/util.py +++ b/pyomo/contrib/mindtpy/util.py @@ -57,10 +57,7 @@ def calc_jacobians(constraint_list, differentiate_mode): # Map nonlinear_constraint --> Map( # variable --> jacobian of constraint w.r.t. variable) jacobians = ComponentMap() - if differentiate_mode == 'reverse_symbolic': - mode = EXPR.differentiate.Modes.reverse_symbolic - elif differentiate_mode == 'sympy': - mode = EXPR.differentiate.Modes.sympy + mode = EXPR.differentiate.Modes(differentiate_mode) for c in constraint_list: vars_in_constr = list(EXPR.identify_variables(c.body)) jac_list = EXPR.differentiate(c.body, wrt_list=vars_in_constr, mode=mode) From 0d39b5d0f20e35c88755dd6557dca63eb1fbf473 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 13 Feb 2024 13:43:35 -0700 Subject: [PATCH 101/103] Update NLv2 to only raise exception on empty models in the legacy API --- pyomo/repn/plugins/nl_writer.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pyomo/repn/plugins/nl_writer.py b/pyomo/repn/plugins/nl_writer.py index 2d5eae151b0..cd570a8a0e1 100644 --- a/pyomo/repn/plugins/nl_writer.py +++ b/pyomo/repn/plugins/nl_writer.py @@ -346,6 +346,17 @@ def __call__(self, model, filename, solver_capability, io_options): row_fname ) as ROWFILE, _open(col_fname) as COLFILE: info = self.write(model, FILE, ROWFILE, COLFILE, config=config) + if not info.variables: + # This exception is included for compatibility with the + # original NL writer v1. + os.remove(filename) + os.remove(row_filename) + os.remove(col_filename) + raise ValueError( + "No variables appear in the Pyomo model constraints or" + " objective. This is not supported by the NL file interface" + ) + # Historically, the NL writer communicated the external function # libraries back to the ASL interface through the PYOMO_AMPLFUNC # environment variable. @@ -854,13 +865,6 @@ def write(self, model): con_vars = con_vars_linear | con_vars_nonlinear all_vars = con_vars | obj_vars n_vars = len(all_vars) - if n_vars < 1: - # TODO: Remove this. This exception is included for - # compatibility with the original NL writer v1. - raise ValueError( - "No variables appear in the Pyomo model constraints or" - " objective. This is not supported by the NL file interface" - ) continuous_vars = set() binary_vars = set() From 9764b84541859c797f7790e3b8475a9e7e7abe24 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 13 Feb 2024 15:35:19 -0700 Subject: [PATCH 102/103] bugfix: fix symbol name --- pyomo/repn/plugins/nl_writer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/repn/plugins/nl_writer.py b/pyomo/repn/plugins/nl_writer.py index cd570a8a0e1..e12c1f47eb1 100644 --- a/pyomo/repn/plugins/nl_writer.py +++ b/pyomo/repn/plugins/nl_writer.py @@ -350,8 +350,8 @@ def __call__(self, model, filename, solver_capability, io_options): # This exception is included for compatibility with the # original NL writer v1. os.remove(filename) - os.remove(row_filename) - os.remove(col_filename) + os.remove(row_fname) + os.remove(col_fname) raise ValueError( "No variables appear in the Pyomo model constraints or" " objective. This is not supported by the NL file interface" From 1462403273125e3cda69720460a0e36240d4c73c Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 13 Feb 2024 15:37:49 -0700 Subject: [PATCH 103/103] add guard for symbolic_solver_labels=False --- pyomo/repn/plugins/nl_writer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyomo/repn/plugins/nl_writer.py b/pyomo/repn/plugins/nl_writer.py index e12c1f47eb1..cda4ee011d3 100644 --- a/pyomo/repn/plugins/nl_writer.py +++ b/pyomo/repn/plugins/nl_writer.py @@ -350,8 +350,9 @@ def __call__(self, model, filename, solver_capability, io_options): # This exception is included for compatibility with the # original NL writer v1. os.remove(filename) - os.remove(row_fname) - os.remove(col_fname) + if config.symbolic_solver_labels: + os.remove(row_fname) + os.remove(col_fname) raise ValueError( "No variables appear in the Pyomo model constraints or" " objective. This is not supported by the NL file interface"