From 1c96382562301f6476e96481bcfa1a0e267274cb Mon Sep 17 00:00:00 2001 From: Darryl Melander Date: Tue, 11 Apr 2023 16:14:06 -0600 Subject: [PATCH 1/2] Add add_column() method to APPSI solvers. This method adds a variable along with its coefficients in the objective and in constraints, all in a single call. Provides the opportunity for more efficient changes to an existing model. --- pyomo/contrib/appsi/base.py | 157 +++++++++++++++++++++++--- pyomo/contrib/appsi/solvers/cbc.py | 9 ++ pyomo/contrib/appsi/solvers/cplex.py | 9 ++ pyomo/contrib/appsi/solvers/gurobi.py | 37 ++++++ pyomo/contrib/appsi/solvers/highs.py | 34 ++++++ pyomo/contrib/appsi/solvers/ipopt.py | 9 ++ 6 files changed, 242 insertions(+), 13 deletions(-) diff --git a/pyomo/contrib/appsi/base.py b/pyomo/contrib/appsi/base.py index ca7255d5628..5010106f79a 100644 --- a/pyomo/contrib/appsi/base.py +++ b/pyomo/contrib/appsi/base.py @@ -803,6 +803,34 @@ def add_constraints(self, cons: List[_GeneralConstraintData]): def add_block(self, block: _BlockData): pass + @abc.abstractmethod + def add_column( + self, + var: _GeneralVarData, + obj_coef: float, + constraints: List[_GeneralConstraintData], + coefficients: List[float], + ): + """Add a column to the solver's model + + Add the Pyomo variable var to the solver's model, and put the + coefficients on the associated constraints in the solver model. + If the obj_coef is not zero, add obj_coef*var to the objective + of the solver's model. + + The column will have already been added to the Pyomo model before + this method is called, such that the variable is already incorporated + into the passed in constraints. + + Parameters + ---------- + var: Var (scalar Var or single _VarData) + obj_coef: float + constraints: list of solver constraints + coefficients: list of coefficients to put on var in the associated constraint + """ + pass + @abc.abstractmethod def remove_variables(self, variables: List[_GeneralVarData]): pass @@ -948,21 +976,18 @@ def _add_variables(self, variables: List[_GeneralVarData]): def add_variables(self, variables: List[_GeneralVarData]): for v in variables: - if id(v) in self._referenced_variables: - raise ValueError( - 'variable {name} has already been added'.format(name=v.name) - ) - self._referenced_variables[id(v)] = [dict(), dict(), None] - self._vars[id(v)] = ( - v, - v._lb, - v._ub, - v.fixed, - v.domain.get_interval(), - v.value, - ) + self._cache_variable_data(v) self._add_variables(variables) + def _cache_variable_data(self, variable: _GeneralVarData): + v = variable + if id(v) in self._referenced_variables: + raise ValueError( + 'variable {name} has already been added'.format(name=v.name) + ) + self._referenced_variables[id(v)] = [dict(), dict(), None] + self._vars[id(v)] = (v, v._lb, v._ub, v.fixed, v.domain.get_interval(), v.value) + @abc.abstractmethod def _add_params(self, params: List[_ParamData]): pass @@ -1042,6 +1067,43 @@ def add_sos_constraints(self, cons: List[_SOSConstraintData]): self._referenced_variables[id(v)][1][con] = None self._add_sos_constraints(cons) + def _add_column( + self, + var: _GeneralVarData, + obj_coef: float, + constraints: List[_GeneralConstraintData], + coefficients: List[float], + ): + """Add a column to the solver's model + + Add the Pyomo variable var to the solver's model, and put the + coefficients on the associated constraints in the solver model. + If the obj_coef is not zero, add obj_coef*var to the objective + of the solver's model. + + The column will have already been added to the Pyomo model before + this method is called, such that the variable is already incorporated + into the passed in constraints. + + Parameters + ---------- + var: Var (scalar Var or single _VarData) + obj_coef: float + constraints: list of solver constraints + coefficients: list of coefficients to put on var in the associated constraint + """ + + # This method is intended to be overridden by derived classes when + # it can be done more efficiently than the default implementation found + # here. The default implementation replaces the existing objective and + # constraints. + self._add_variables([var]) + if obj_coef != 0.0: + self._set_objective(self._objective) + # Constraint objects have already been updated, just replace them + self._remove_constraints(constraints) + self._add_constraints(constraints) + @abc.abstractmethod def _set_objective(self, obj: _GeneralObjectiveData): pass @@ -1463,6 +1525,75 @@ def update(self, timer: HierarchicalTimer = None): self.set_objective(pyomo_obj) timer.stop('objective') + def add_column( + self, + var: _GeneralVarData, + obj_coef: float, + constraints: List[_GeneralConstraintData], + coefficients: List[float], + timer: HierarchicalTimer = None, + ): + """Add a column to Pyomo model and solver model. + + This method takes a Pyomo variable var that has already been + added to the Pyomo model and adds it to the Pyomo model's objective + and constraints. It will then add the variable to the solver's model, + including changes to the objective and constraints. + + Parameters + ---------- + model: pyomo ConcreteModel to which the column will be added + var: Var (scalar Var or single _VarData) + obj_coef: float, pyo.Param + constraints: list of scalar Constraints of single _ConstraintDatas + coefficients: list of the coefficient to put on var in the associated constraint + + """ + if timer is None: + timer = HierarchicalTimer() + + timer.start('add column') + + # First we update the Pyomo model and the PersistentBase data + timer.start('update Pyomo model') + + # Add the column's data to cached model info. + # We do this before calling update() so that the variable + # won't be added to the solver model at this time. + self._cache_variable_data(var) + vid = id(var) + + # Make sure the model is up to date before adding the new column + self.update(timer) + + # Add the variable to the objective + if obj_coef != 0.0 and self._objective is not None: + # Add variable to the pyomo objective + self._objective.expr = self._objective.expr + obj_coef * var + # Update cache + self._objective_expr = self._objective.expr + self._referenced_variables[vid][2] = self._objective + self._vars_referenced_by_obj.append(var) + + # Add the variable to each constraint + for con, coef in zip(constraints, coefficients): + if coef != 0.0: + # Add variable to the pyomo constraint + con.set_value((con.lower, con.body + coef * var, con.upper)) + # Update cache + self._active_constraints[con] = (con.lower, con.body, con.upper) + self._referenced_variables[vid][0][con] = None + self._vars_referenced_by_con[con].append(var) + + timer.stop('update Pyomo model') + + # Add the new column to the solver model + timer.start('update solver model') + self._add_column(var, obj_coef, constraints, coefficients) + timer.stop('update solver model') + + timer.stop('add column') + legacy_termination_condition_map = { TerminationCondition.unknown: LegacyTerminationCondition.unknown, diff --git a/pyomo/contrib/appsi/solvers/cbc.py b/pyomo/contrib/appsi/solvers/cbc.py index b31a96dbf8a..747cfc3a8bf 100644 --- a/pyomo/contrib/appsi/solvers/cbc.py +++ b/pyomo/contrib/appsi/solvers/cbc.py @@ -165,6 +165,15 @@ def add_constraints(self, cons: List[_GeneralConstraintData]): def add_block(self, block: _BlockData): self._writer.add_block(block) + def add_column( + self, + var: _GeneralVarData, + obj_coef: float, + constraints: List[_GeneralConstraintData], + coefficients: List[float], + ): + self._writer.add_column(var, obj_coef, constraints, coefficients) + def remove_variables(self, variables: List[_GeneralVarData]): self._writer.remove_variables(variables) diff --git a/pyomo/contrib/appsi/solvers/cplex.py b/pyomo/contrib/appsi/solvers/cplex.py index 6c5e281ffac..3e9944e3621 100644 --- a/pyomo/contrib/appsi/solvers/cplex.py +++ b/pyomo/contrib/appsi/solvers/cplex.py @@ -180,6 +180,15 @@ def add_constraints(self, cons: List[_GeneralConstraintData]): def add_block(self, block: _BlockData): self._writer.add_block(block) + def add_column( + self, + var: _GeneralVarData, + obj_coef: float, + constraints: List[_GeneralConstraintData], + coefficients: List[float], + ): + self._writer.add_column(var, obj_coef, constraints, coefficients) + def remove_variables(self, variables: List[_GeneralVarData]): self._writer.remove_variables(variables) diff --git a/pyomo/contrib/appsi/solvers/gurobi.py b/pyomo/contrib/appsi/solvers/gurobi.py index a2324816e3b..241055f6629 100644 --- a/pyomo/contrib/appsi/solvers/gurobi.py +++ b/pyomo/contrib/appsi/solvers/gurobi.py @@ -857,6 +857,43 @@ def _set_objective(self, obj): self._solver_model.setObjective(gurobi_expr + value(repn_constant), sense=sense) self._needs_updated = True + def _add_column( + self, + var: _GeneralVarData, + obj_coef: float, + constraints: List[_GeneralConstraintData], + coefficients: List[float], + ): + # Get variable data in solver format + varname = self._symbol_map.getSymbol(var, self._labeler) + mutable_lb = dict() + mutable_ub = dict() + lb, ub, vtype = self._process_domain_and_bounds( + var, id(var), mutable_lb, mutable_ub, 0, None + ) + + # Map pyomo constraints to solver constraints + solver_cons = [self._pyomo_con_to_solver_con_map[con] for con in constraints] + + # Add the variable to the solver, including constraints + gurobipy_var = self._solver_model.addVar( + obj=obj_coef, + lb=lb, + ub=ub, + vtype=vtype, + name=varname, + column=gurobipy.Column(coeffs=coefficients, constrs=solver_cons), + ) + + # Do book keeping + self._pyomo_var_to_solver_var_map[id(var)] = gurobipy_var + if 0 in mutable_lb: + mutable_lb[0].var = gurobipy_var + if 0 in mutable_ub: + mutable_ub[0].var = gurobipy_var + self._vars_added_since_update.add(var) + self._needs_updated = True + def _postsolve(self, timer: HierarchicalTimer): config = self.config diff --git a/pyomo/contrib/appsi/solvers/highs.py b/pyomo/contrib/appsi/solvers/highs.py index 9de5accfb91..5919b2906ad 100644 --- a/pyomo/contrib/appsi/solvers/highs.py +++ b/pyomo/contrib/appsi/solvers/highs.py @@ -441,6 +441,40 @@ def _add_sos_constraints(self, cons: List[_SOSConstraintData]): 'Highs interface does not support SOS constraints' ) + def _add_column( + self, + var: _GeneralVarData, + obj_coef: float, + constraints: List[_GeneralConstraintData], + coefficients: List[float], + ): + self._sol = None + if self._last_results_object is not None: + self._last_results_object.solution_loader.invalidate() + + # Get variable data + v_id = id(var) + lb, ub, vtype = self._process_domain_and_bounds(v_id) + + # prepare constraint data + solver_cons = np.fromiter( + (self._pyomo_con_to_solver_con_map[con] for con in constraints), + dtype=np.int32, + ) + solver_coefs = np.array(coefficients, dtype=np.double) + + # Add the column + self._solver_model.addCol( + obj_coef, lb, ub, len(constraints), solver_cons, solver_coefs + ) + + # Add new variable to the variable map + var_idx = len(self._pyomo_var_to_solver_var_map) + self._pyomo_var_to_solver_var_map[v_id] = var_idx + + # Set the variable type + self._solver_model.changeColIntegrality(var_idx, vtype) + def _remove_constraints(self, cons: List[_GeneralConstraintData]): self._sol = None if self._last_results_object is not None: diff --git a/pyomo/contrib/appsi/solvers/ipopt.py b/pyomo/contrib/appsi/solvers/ipopt.py index fde4c55073d..8886f6dc79f 100644 --- a/pyomo/contrib/appsi/solvers/ipopt.py +++ b/pyomo/contrib/appsi/solvers/ipopt.py @@ -228,6 +228,15 @@ def add_constraints(self, cons: List[_GeneralConstraintData]): def add_block(self, block: _BlockData): self._writer.add_block(block) + def add_column( + self, + var: _GeneralVarData, + obj_coef: float, + constraints: List[_GeneralConstraintData], + coefficients: List[float], + ): + self._writer.add_column(var, obj_coef, constraints, coefficients) + def remove_variables(self, variables: List[_GeneralVarData]): self._writer.remove_variables(variables) From 3586d8f7771bf822ae82665b65a6c6389e5e135b Mon Sep 17 00:00:00 2001 From: Darryl Melander Date: Tue, 11 Apr 2023 16:14:20 -0600 Subject: [PATCH 2/2] Tests for APPSI add_column() --- .../solvers/tests/test_persistent_solvers.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pyomo/contrib/appsi/solvers/tests/test_persistent_solvers.py b/pyomo/contrib/appsi/solvers/tests/test_persistent_solvers.py index bafccb3527c..c9b07e5cd70 100644 --- a/pyomo/contrib/appsi/solvers/tests/test_persistent_solvers.py +++ b/pyomo/contrib/appsi/solvers/tests/test_persistent_solvers.py @@ -9,6 +9,7 @@ from pyomo.contrib.appsi.solvers import Gurobi, Ipopt, Cplex, Cbc, Highs from typing import Type from pyomo.core.expr.numeric_expr import LinearExpression +from pyomo.core.expr.compare import assertExpressionsEqual import os numpy, numpy_available = attempt_import('numpy') @@ -1196,6 +1197,40 @@ def test_bug_1(self, name: str, opt_class: Type[PersistentSolver]): self.assertEqual(res.termination_condition, TerminationCondition.optimal) self.assertAlmostEqual(res.best_feasible_objective, 3) + @parameterized.expand(input=all_solvers) + def test_add_column(self, name: str, opt_class: Type[PersistentSolver]): + '''Verify that add_column() modifies models correctly.''' + opt: PersistentSolver = opt_class() + if not opt.available(): + raise unittest.SkipTest + + # Create a concrete model + m = pe.ConcreteModel() + m.x = pe.Var(within=pe.NonNegativeReals) + m.c1 = pe.Constraint(expr=(0, m.x, 1)) + m.c2 = pe.Constraint(expr=(0, m.x, 1)) + m.obj = pe.Objective(expr=-m.x) + + # Solve first version of concrete model + opt.solve(m) + self.assertAlmostEqual(m.x.value, 1) + + # Add a new variable + m.y = pe.Var(within=pe.NonNegativeReals) + + # Add new column for the new variable + opt.add_column(m.y, -3, [m.c2], [2]) + + # verify the pyomo model was updated correctly + # assertExpressionsEqual(self, m.obj.expr, -m.x - 3 * m.y) + # assertExpressionsEqual(self, m.c1.expr, pe.Constraint(expr=(0, m.x, 1)).expr) + # assertExpressionsEqual(self, m.c2.expr, pe.Constraint(expr=(0, m.x + 2 * m.y, 1)).expr) + + # Re-solve and verify results reflect the additional column + opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 0.5) + @unittest.skipUnless(cmodel_available, 'appsi extensions are not available') class TestLegacySolverInterface(unittest.TestCase):