diff --git a/doc/OnlineDocs/modeling_extensions/gdp/modeling.rst b/doc/OnlineDocs/modeling_extensions/gdp/modeling.rst index 3a666e63233..671575d5775 100644 --- a/doc/OnlineDocs/modeling_extensions/gdp/modeling.rst +++ b/doc/OnlineDocs/modeling_extensions/gdp/modeling.rst @@ -160,11 +160,10 @@ Usage: >>> m.Y = BooleanVar(m.my_set) >>> m.p = LogicalConstraint(expr=atleast(3, m.Y)) >>> m.p.pprint() - p : Size=1, Index=None, Active=False + p : Size=1, Index=None, Active=True Key : Body : Active - None : atleast(3: [Y[1], Y[2], Y[3], Y[4]]) : False + None : atleast(3: [Y[1], Y[2], Y[3], Y[4]]) : True >>> TransformationFactory('core.logical_to_linear').apply_to(m) - ... >>> # constraint auto-generated by transformation >>> m.logic_to_linear.transformed_constraints.pprint() transformed_constraints : Size=1, Index=logic_to_linear.transformed_constraints_index, Active=True diff --git a/doc/OnlineDocs/pyomo_modeling_components/Suffixes.rst b/doc/OnlineDocs/pyomo_modeling_components/Suffixes.rst index 58c4de9fbc2..e45fe2d74b7 100644 --- a/doc/OnlineDocs/pyomo_modeling_components/Suffixes.rst +++ b/doc/OnlineDocs/pyomo_modeling_components/Suffixes.rst @@ -388,6 +388,7 @@ Suffix component with an IMPORT_EXPORT direction. ipopt = pyo.SolverFactory('ipopt') + The difference in performance can be seen by examining Ipopt's iteration log with and without warm starting: @@ -405,13 +406,9 @@ log with and without warm starting: iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls 0 1.6109693e+01 1.12e+01 5.28e-01 -1.0 0.00e+00 - 0.00e+00 0.00e+00 0 1 1.6982239e+01 7.30e-01 1.02e+01 -1.0 6.11e-01 - 7.19e-02 1.00e+00f 1 - 2 1.7318411e+01 3.60e-02 5.05e-01 -1.0 1.61e-01 - 1.00e+00 1.00e+00h 1 - 3 1.6849424e+01 2.78e-01 6.68e-02 -1.7 2.85e-01 - 7.94e-01 1.00e+00h 1 - 4 1.7051199e+01 4.71e-03 2.78e-03 -1.7 6.06e-02 - 1.00e+00 1.00e+00h 1 - 5 1.7011979e+01 7.19e-03 8.50e-03 -3.8 3.66e-02 - 9.45e-01 9.98e-01h 1 - 6 1.7014271e+01 1.74e-05 9.78e-06 -3.8 3.33e-03 - 1.00e+00 1.00e+00h 1 - 7 1.7014021e+01 1.23e-07 1.82e-07 -5.7 2.69e-04 - 1.00e+00 1.00e+00h 1 - 8 1.7014017e+01 1.77e-11 2.52e-11 -8.6 3.32e-06 - 1.00e+00 1.00e+00h 1 + 2 1.7318411e+01 ... + ... + 8 1.7014017e+01 ... Number of Iterations....: 8 ... @@ -441,7 +438,7 @@ log with and without warm starting: iter objective inf_pr inf_du lg(mu) ||d|| lg(rg) alpha_du alpha_pr ls 0 1.7014032e+01 2.00e-06 4.07e-06 -6.0 0.00e+00 - 0.00e+00 0.00e+00 0 1 1.7014019e+01 3.65e-12 1.00e-11 -6.0 2.50e-01 - 1.00e+00 1.00e+00h 1 - 2 1.7014017e+01 4.48e-12 6.42e-12 -9.0 1.92e-06 - 1.00e+00 1.00e+00h 1 + 2 1.7014017e+01 ... Number of Iterations....: 2 ... diff --git a/examples/pyomobook/abstract-ch/pyomo.abstract7.txt b/examples/pyomobook/abstract-ch/pyomo.abstract7.txt index 9d669b54b39..afb8b7fae21 100644 --- a/examples/pyomobook/abstract-ch/pyomo.abstract7.txt +++ b/examples/pyomobook/abstract-ch/pyomo.abstract7.txt @@ -13,7 +13,7 @@ model: save file: None save format: None symbolic solver labels: false - file determinism: 1 + file determinism: None transform: [] preprocess: [] runtime: diff --git a/examples/pyomobook/abstract-ch/pyomo.model3.sh b/examples/pyomobook/abstract-ch/pyomo.model3.sh index 94d1a4b500d..ebba20c075b 100755 --- a/examples/pyomobook/abstract-ch/pyomo.model3.sh +++ b/examples/pyomobook/abstract-ch/pyomo.model3.sh @@ -3,5 +3,5 @@ # @cmd: pyomo convert --output=concrete1.nl concrete1.py # @:cmd -diff concrete1.nl concrete1-ref.nl +python -m pyomo.repn.tests.ampl.nl_diff concrete1.nl concrete1-ref.nl rm -f results.yml results.json concrete1.nl diff --git a/pyomo/contrib/pynumero/algorithms/solvers/tests/test_pyomo_ext_cyipopt.py b/pyomo/contrib/pynumero/algorithms/solvers/tests/test_pyomo_ext_cyipopt.py index 6eb276c473d..67861fc7054 100644 --- a/pyomo/contrib/pynumero/algorithms/solvers/tests/test_pyomo_ext_cyipopt.py +++ b/pyomo/contrib/pynumero/algorithms/solvers/tests/test_pyomo_ext_cyipopt.py @@ -147,7 +147,7 @@ def test_pyomo_external_model_scaling(self): inputs=[m.Pin, m.c1, m.c2, m.F], outputs=[m.P1, m.P2], outputs_eqn_scaling=[10.0, 11.0], - nl_file_options={'file_determinism': 3}, + nl_file_options={'file_determinism': 2}, ) # solve the problem @@ -220,7 +220,7 @@ def test_pyomo_external_model_ndarray_scaling(self): inputs=[m.Pin, m.c1, m.c2, m.F], outputs=[m.P1, m.P2], outputs_eqn_scaling=np.asarray([10.0, 11.0], dtype=np.float64), - nl_file_options={'file_determinism': 3}, + nl_file_options={'file_determinism': 2}, ) # solve the problem diff --git a/pyomo/contrib/sensitivity_toolbox/tests/test_sens_unit.py b/pyomo/contrib/sensitivity_toolbox/tests/test_sens_unit.py index b0560f1723d..eda2d6b9fb4 100644 --- a/pyomo/contrib/sensitivity_toolbox/tests/test_sens_unit.py +++ b/pyomo/contrib/sensitivity_toolbox/tests/test_sens_unit.py @@ -698,9 +698,16 @@ def test_get_dsdp1(self): getattr(m, v).setlb(theta[v]) getattr(m, v).setub(theta[v]) dsdp, col = get_dsdp(m, variable_name, theta) - np.testing.assert_almost_equal(dsdp.toarray(),[[1., 0., 1., 0.],[0., 1., 0., 1.]]) - - assert col == ['x1', 'x2', 'p1', 'p2'] + ref = { + 'x1': [1., 0.], + 'x2': [0., 1.], + 'p1': [1., 0.], + 'p2': [0., 1.], + } + np.testing.assert_almost_equal( + dsdp.toarray(), + np.vstack(ref[c] for c in col).transpose() + ) @unittest.skipIf(not opt_kaug.available(False), "k_aug is not available") @unittest.skipIf(not opt_ipopt.available(False), "ipopt is not available") @@ -757,10 +764,26 @@ def test_get_dfds_dcds(self): getattr(m, v).setlb(theta[v]) getattr(m, v).setub(theta[v]) gradient_f, gradient_c, col ,row, line_dic= get_dfds_dcds(m, variable_name) - np.testing.assert_almost_equal( gradient_f, [10., 50., 15., 35.]) - np.testing.assert_almost_equal( gradient_c.toarray(), [[ 1., 0., -1., 0.], [ 0., 1., 0., -1.]]) - assert col == ['x1', 'x2', 'p1', 'p2'] - assert row == ['c1', 'c2', 'obj'] + + ref_f = { + 'x1': [10.], + 'x2': [50.], + 'p1': [15.], + 'p2': [35.], + } + ref_c = { + 'x1': [1., 0.], + 'x2': [0., 1.], + 'p1': [-1., 0.], + 'p2': [0., -1.], + } + np.testing.assert_almost_equal( + gradient_f, np.hstack(ref_f[v] for v in col) + ) + np.testing.assert_almost_equal( + gradient_c.toarray(), + np.vstack(ref_c[v] for v in col).transpose() + ) @unittest.skipIf(not opt_kaug.available(False), "k_aug is not available") @unittest.skipIf(not opt_dotsens.available(False), "dot_sens is not available") diff --git a/pyomo/mpec/complementarity.py b/pyomo/mpec/complementarity.py index 6d264af8fae..3cc7d2b8a10 100644 --- a/pyomo/mpec/complementarity.py +++ b/pyomo/mpec/complementarity.py @@ -77,10 +77,11 @@ def to_standard_form(self): # the form: # l1 <= v1 <= u1 OR l2 <= v2 <= u2 # - # Note that this transformation creates more variables and constraints - # than are strictly necessary. However, we don't have a complete list of - # the variables used in a model's complementarity conditions when adding - # a single condition, so we add additional variables. + # Note that this transformation creates more variables and + # constraints than are strictly necessary. However, we don't + # have a complete list of the variables used in a model's + # complementarity conditions when adding a single condition, so + # we add additional variables. # # This has the form: # @@ -144,14 +145,14 @@ def set_value(self, cc): # The ComplementarityTuple has a fixed length, so we initialize # the _args component and return # - self._args = ( as_numeric(cc.arg0), as_numeric(cc.arg1) ) + self._args = ( cc.arg0, cc.arg1 ) # elif cc.__class__ is tuple: if len(cc) != 2: raise ValueError( "Invalid tuple for Complementarity %s (expected 2-tuple):" "\n\t%s" % (self.name, cc) ) - self._args = tuple( as_numeric(x) for x in cc ) + self._args = cc elif cc is Complementarity.Skip: del self.parent_component()[self.index()] elif cc.__class__ is list: diff --git a/pyomo/mpec/tests/t11_nlxfrm.nl b/pyomo/mpec/tests/t11_nlxfrm.nl index 3a3b18a1a93..f8888c4b96f 100644 --- a/pyomo/mpec/tests/t11_nlxfrm.nl +++ b/pyomo/mpec/tests/t11_nlxfrm.nl @@ -1,5 +1,5 @@ g3 1 1 0 # problem unknown - 3 1 0 0 1 # vars, constraints, objectives, ranges, eqns + 3 1 0 0 1 # vars, constraints, objectives, ranges, eqns; KNOWN BUG WITH NLv1 WRTIER!!! 0 0 1 0 0 0 # nonlinear constrs, objs; ccons: lin, nonlin, nd, nzlb 0 0 # network constraints: nonlinear, linear 0 0 0 # nonlinear vars in constraints, objectives, both diff --git a/pyomo/mpec/tests/t11_nlxfrm.nl_v2 b/pyomo/mpec/tests/t11_nlxfrm.nl_v2 new file mode 100644 index 00000000000..8038ffe6d15 --- /dev/null +++ b/pyomo/mpec/tests/t11_nlxfrm.nl_v2 @@ -0,0 +1,31 @@ +g3 1 1 0 # problem unknown + 3 2 0 0 1 # vars, constraints, objectives, ranges, eqns + 0 0 1 0 0 0 # nonlinear constrs, objs; ccons: lin, nonlin, nd, nzlb + 0 0 # network constraints: nonlinear, linear + 0 0 0 # nonlinear vars in constraints, objectives, both + 0 0 0 1 # linear network variables; functions; arith, flags + 0 0 0 0 0 # discrete variables: binary, integer, nonlinear (b,c,o) + 4 0 # nonzeros in Jacobian, obj. gradient + 0 0 # max name lengths: constraints, variables + 0 0 0 0 0 # common exprs: b,c,o,c1,o1 +C0 +n0 +C1 +n0 +x0 +r +5 0 2 +4 2.0 +b +3 +3 +3 +k2 +1 +2 +J0 1 +2 1 +J1 3 +0 1 +1 1 +2 1 diff --git a/pyomo/mpec/tests/t12_nlxfrm.nl b/pyomo/mpec/tests/t12_nlxfrm.nl index 3a3b18a1a93..f8888c4b96f 100644 --- a/pyomo/mpec/tests/t12_nlxfrm.nl +++ b/pyomo/mpec/tests/t12_nlxfrm.nl @@ -1,5 +1,5 @@ g3 1 1 0 # problem unknown - 3 1 0 0 1 # vars, constraints, objectives, ranges, eqns + 3 1 0 0 1 # vars, constraints, objectives, ranges, eqns; KNOWN BUG WITH NLv1 WRTIER!!! 0 0 1 0 0 0 # nonlinear constrs, objs; ccons: lin, nonlin, nd, nzlb 0 0 # network constraints: nonlinear, linear 0 0 0 # nonlinear vars in constraints, objectives, both diff --git a/pyomo/mpec/tests/t12_nlxfrm.nl_v2 b/pyomo/mpec/tests/t12_nlxfrm.nl_v2 new file mode 100644 index 00000000000..8038ffe6d15 --- /dev/null +++ b/pyomo/mpec/tests/t12_nlxfrm.nl_v2 @@ -0,0 +1,31 @@ +g3 1 1 0 # problem unknown + 3 2 0 0 1 # vars, constraints, objectives, ranges, eqns + 0 0 1 0 0 0 # nonlinear constrs, objs; ccons: lin, nonlin, nd, nzlb + 0 0 # network constraints: nonlinear, linear + 0 0 0 # nonlinear vars in constraints, objectives, both + 0 0 0 1 # linear network variables; functions; arith, flags + 0 0 0 0 0 # discrete variables: binary, integer, nonlinear (b,c,o) + 4 0 # nonzeros in Jacobian, obj. gradient + 0 0 # max name lengths: constraints, variables + 0 0 0 0 0 # common exprs: b,c,o,c1,o1 +C0 +n0 +C1 +n0 +x0 +r +5 0 2 +4 2.0 +b +3 +3 +3 +k2 +1 +2 +J0 1 +2 1 +J1 3 +0 1 +1 1 +2 1 diff --git a/pyomo/mpec/tests/t2a_nlxfrm.nl_v2 b/pyomo/mpec/tests/t2a_nlxfrm.nl_v2 new file mode 100644 index 00000000000..3d4bfb62b7a --- /dev/null +++ b/pyomo/mpec/tests/t2a_nlxfrm.nl_v2 @@ -0,0 +1,42 @@ +g3 1 1 0 # problem unknown + 5 3 0 0 2 # vars, constraints, objectives, ranges, eqns + 0 0 1 0 0 0 # nonlinear constrs, objs; ccons: lin, nonlin, nd, nzlb + 0 0 # network constraints: nonlinear, linear + 0 0 0 # nonlinear vars in constraints, objectives, both + 0 0 0 1 # linear network variables; functions; arith, flags + 0 0 0 0 0 # discrete variables: binary, integer, nonlinear (b,c,o) + 7 0 # nonzeros in Jacobian, obj. gradient + 0 0 # max name lengths: constraints, variables + 0 0 0 0 0 # common exprs: b,c,o,c1,o1 +C0 +n0 +C1 +n0 +C2 +n0 +x0 +r +5 2 5 +4 0.0 +4 0.0 +b +3 +3 +3 +3 +1 -1 +k4 +1 +3 +4 +6 +J0 1 +3 1 +J1 3 +0 1 +1 1 +3 1 +J2 3 +1 -1 +2 1 +4 1 diff --git a/pyomo/mpec/tests/t2b_nlxfrm.nl_v2 b/pyomo/mpec/tests/t2b_nlxfrm.nl_v2 new file mode 100644 index 00000000000..3b16b9b8d00 --- /dev/null +++ b/pyomo/mpec/tests/t2b_nlxfrm.nl_v2 @@ -0,0 +1,42 @@ +g3 1 1 0 # problem unknown + 5 3 0 0 2 # vars, constraints, objectives, ranges, eqns + 0 0 1 0 0 0 # nonlinear constrs, objs; ccons: lin, nonlin, nd, nzlb + 0 0 # network constraints: nonlinear, linear + 0 0 0 # nonlinear vars in constraints, objectives, both + 0 0 0 1 # linear network variables; functions; arith, flags + 0 0 0 0 0 # discrete variables: binary, integer, nonlinear (b,c,o) + 7 0 # nonzeros in Jacobian, obj. gradient + 0 0 # max name lengths: constraints, variables + 0 0 0 0 0 # common exprs: b,c,o,c1,o1 +C0 +n0 +C1 +n0 +C2 +n0 +x0 +r +5 1 5 +4 -1.0 +4 0.0 +b +3 +3 +3 +3 +2 0 +k4 +1 +3 +4 +6 +J0 1 +3 1 +J1 3 +1 1 +2 -1 +3 1 +J2 3 +0 -1 +1 -1 +4 1 diff --git a/pyomo/mpec/tests/test_complementarity.py b/pyomo/mpec/tests/test_complementarity.py index ada1203ac7c..0534b425441 100644 --- a/pyomo/mpec/tests/test_complementarity.py +++ b/pyomo/mpec/tests/test_complementarity.py @@ -31,6 +31,7 @@ from pyomo.gdp import Disjunct, Disjunction from pyomo.mpec import Complementarity, complements, ComplementarityList from pyomo.opt import ProblemFormat +from pyomo.repn.plugins.nl_writer import FileDeterminism from pyomo.repn.tests.ampl.nl_diff import load_and_compare_nl_baseline currdir = this_file_dir() @@ -391,16 +392,32 @@ class CCTests_simple_disjunction(CCTests, unittest.TestCase): xfrm = 'mpec.simple_disjunction' -class CCTests_nl_nlxfrm(CCTests, unittest.TestCase): - +class CCTests_nl_nlxfrm(CCTests): def _test(self, tname, M): bfile = os.path.join(currdir, tname + '_nlxfrm.nl') xfrm = TransformationFactory('mpec.nl') xfrm.apply_to(M) + fd = FileDeterminism.SORT_INDICES if self._nl_version == 'nl_v2' else 1 with TempfileManager: ofile = TempfileManager.create_tempfile(suffix='_nlxfrm.out') - M.write(ofile, format=ProblemFormat.nl) - self.assertEqual(*load_and_compare_nl_baseline(bfile, ofile)) + M.write( + ofile, + format=self._nl_version, + io_options={ + 'symbolic_solver_labels': False, + 'file_determinism': fd, + } + ) + self.assertEqual(*load_and_compare_nl_baseline( + bfile, ofile, self._nl_version)) + + +class CCTests_nl_nlxfrm_nlv1(CCTests_nl_nlxfrm, unittest.TestCase): + _nl_version = 'nl_v1' + + +class CCTests_nl_nlxfrm_nlv2(CCTests_nl_nlxfrm, unittest.TestCase): + _nl_version = 'nl_v2' class DescendIntoDisjunct(unittest.TestCase): @@ -453,9 +470,6 @@ def test_simple_nonlinear_descend_into_disjunct(self): def test_simple_nonlinear_on_disjunct(self): m = self.get_model() - TransformationFactory('mpec.simple_nonlinear').apply_to(m.disjunct1) - self.check_simple_nonlinear(m) - def check_standard_form(self, m): # check that we have what we expect on disjunct1 compBlock = m.disjunct1.component('comp') diff --git a/pyomo/repn/plugins/__init__.py b/pyomo/repn/plugins/__init__.py index 9aacacd69fd..e690cb9ee39 100644 --- a/pyomo/repn/plugins/__init__.py +++ b/pyomo/repn/plugins/__init__.py @@ -19,4 +19,4 @@ def load(): from pyomo.opt import WriterFactory WriterFactory.register('nl', 'Generate the corresponding AMPL NL file.')( - WriterFactory.get_class('nl_v1')) + WriterFactory.get_class('nl_v2')) diff --git a/pyomo/repn/plugins/nl_writer.py b/pyomo/repn/plugins/nl_writer.py index 9ea59b88c34..62b4c5a9525 100644 --- a/pyomo/repn/plugins/nl_writer.py +++ b/pyomo/repn/plugins/nl_writer.py @@ -65,6 +65,7 @@ # Feasibility tolerance for trivial (fixed) constraints TOL = 1e-8 inf = float('inf') +minus_inf = -inf nan = float('nan') HALT_ON_EVALUATION_ERROR = False @@ -277,7 +278,7 @@ def __call__(self, model, filename, solver_capability, io_options): _open = lambda fname: open(fname, 'w') else: _open = nullcontext - with open(filename, 'w') as FILE, \ + with open(filename, 'w', newline='') as FILE, \ _open(row_fname) as ROWFILE, \ _open(col_fname) as COLFILE: info = self.write( @@ -587,6 +588,13 @@ def write(self, model): linear_cons = [] n_ranges = 0 n_equality = 0 + n_complementarity_nonlin = 0 + n_complementarity_lin = 0 + # TODO: update the writer to tabulate and report the range and + # nzlb values. Low priority, as they do not appear to be + # required for solvers like PATH. + n_complementarity_range = 0 + n_complementarity_nz_var_lb = 0 for con in model.component_data_objects( Constraint, active=True, sort=sorter): if with_debug_timing and con.parent_component() is not last_parent: @@ -597,10 +605,14 @@ def write(self, model): self._record_named_expression_usage( expr.named_exprs, con, 0) lb = con.lb - if lb is not None: + if lb == minus_inf: + lb = None + elif lb is not None: lb = repr(lb - expr.const) ub = con.ub - if ub is not None: + if ub == inf: + ub = None + elif ub is not None: ub = repr(ub - expr.const) _type = _RANGE_TYPE(lb, ub) if _type == 4: @@ -608,9 +620,23 @@ def write(self, model): elif _type == 0: n_ranges += 1 elif _type == 3: #and self.config.skip_trivial_constraints: - # FIXME: historically the NL writer was - # hard-coded to skip all unbounded constraints continue + pass + # FIXME: this is a HACK to be compatible with the NLv1 + # writer. In the future, this writer should be expanded to + # look for and process Complementarity components (assuming + # that they are in an acceptable form). + if hasattr(con, '_complementarity'): + _type = 5 + # we are going to pass the complementarity type and the + # corresponding variable id() as the "lb" and "ub" for + # the range. + lb = con._complementarity + ub = con._vid + if expr.nonlinear: + n_complementarity_nonlin += 1 + else: + n_complementarity_lin += 1 if expr.nonlinear: constraints.append((con, expr, _type, lb, ub)) elif expr.linear: @@ -828,9 +854,13 @@ def write(self, model): for idx, _id in enumerate(variables): v = var_map[_id] lb, ub = v.bounds - if lb is not None: + if lb == minus_inf: + lb = None + elif lb is not None: lb = repr(lb) - if ub is not None: + if ub == inf: + ub = None + elif ub is not None: ub = repr(ub) variables[idx] = (v, _id, _RANGE_TYPE(lb, ub), lb, ub) timer.toc("Computed variable bounds", level=logging.DEBUG) @@ -942,7 +972,52 @@ def write(self, model): # # LINE 1 # - ostream.write("g3 1 1 0\t# problem %s\n" % (model.name,)) + if (visitor.encountered_string_arguments + and 'b' not in getattr(ostream, 'mode', '') + ): + # Not all streams support tell() + try: + _written_bytes = ostream.tell() + except IOError: + _written_bytes = None + + line_1_txt = f"g3 1 1 0\t# problem {model.name}\n" + ostream.write(line_1_txt) + + # If there were any string arguments, then we need to ensure + # that ostream is not converting newlines to something other + # than '\n'. Binary files do not perform newline mapping (of + # course, we will also need to map all the str to bytes for + # binary-mode I/O). + if (visitor.encountered_string_arguments + and 'b' not in getattr(ostream, 'mode', '') + ): + if _written_bytes is None: + _written_bytes = 0 + else: + _written_bytes = ostream.tell() - _written_bytes + if not _written_bytes: + if os.linesep != '\n': + logger.warning( + "Writing NL file containing string arguments to a " + "text output stream that does not support tell() on " + "a platform with default line endings other than " + "'\\n'. Current versions of the ASL " + "(through at least 20190605) require UNIX-style " + "newlines as terminators for string arguments: " + "it is possible that the ASL may refuse to read " + "the NL file.") + else: + if ostream.encoding: + line_1_txt = line_1_txt.encode(ostream.encoding) + if len(line_1_txt) != _written_bytes: + logger.error( + "Writing NL file containing string arguments to a " + "text output stream with line endings other than '\\n' " + "Current versions of the ASL " + "(through at least 20190605) require UNIX-style " + "newlines as terminators for string arguments.") + # # LINE 2 # @@ -963,10 +1038,15 @@ def write(self, model): "# nonlinear constrs, objs; ccons: lin, nonlin, nd, nzlb\n" % ( n_nonlinear_cons, n_nonlinear_objs, - 0, # ccons_lin, - 0, # ccons_nonlin, - 0, # ccons_nd, - 0, # ccons_nzlb, + # num linear complementarity constraints + n_complementarity_lin, + # num nonlinear complementarity constraints + n_complementarity_nonlin, + # num complementarities involving double inequalities + n_complementarity_range, + # num complemented variables with either a nonzero lower + # bound or any upper bound (excluding ranges) + n_complementarity_nz_var_lb, )) # # LINE 4 @@ -1184,6 +1264,9 @@ def write(self, model): ostream.write(f"2 {info[3]}{row_comments[row_idx]}\n") elif i == 0: # lb <= body <= ub ostream.write(f"0 {info[3]} {info[4]}{row_comments[row_idx]}\n") + elif i == 5: # complementarity + ostream.write(f"5 {info[3]} {1+column_order[info[4]]}" + f"{row_comments[row_idx]}\n") else: # i == 3; unbounded ostream.write(f"3{row_comments[row_idx]}\n") @@ -2102,6 +2185,7 @@ def _before_native(visitor, child): return False, (_CONSTANT, child) def _before_string(visitor, child): + visitor.encountered_string_arguments = True ans = AMPLRepn(child, None, None) ans.nl = (visitor.template.string % (len(child), child), ()) return False, (_GENERAL, ans) @@ -2257,6 +2341,7 @@ def __init__(self, template, subexpression_cache, subexpression_order, self.used_named_expressions = used_named_expressions self.symbolic_solver_labels = symbolic_solver_labels self.use_named_exprs = use_named_exprs + self.encountered_string_arguments = False #self.value_cache = {} def initializeWalker(self, expr): diff --git a/pyomo/repn/tests/ampl/nl_diff.py b/pyomo/repn/tests/ampl/nl_diff.py index 76d6bc99c46..17d5a6c82dd 100644 --- a/pyomo/repn/tests/ampl/nl_diff.py +++ b/pyomo/repn/tests/ampl/nl_diff.py @@ -25,6 +25,7 @@ _strip_comment = re.compile(r'\s*#.*') _norm_negation = re.compile(r'(?m)^o2(\s*#\s*\*)?\nn-1(.0)?\s*\n') _norm_timesone = re.compile(r'(?m)^o2(\s*#\s*\*)?\nn1(.0)?\s*\n') +_norm_double_negation = re.compile(r'(?m)^o16(\s*#\s*-)?\no16(\s*#\s*-)?\n') def _compare_floats(base, test, abstol=1e-14, reltol=1e-14): base = base.split() @@ -67,6 +68,8 @@ def _update_subsets(subset, base, test): def _preprocess_data(data): # Normalize negation (convert " * -1" to the negation operator) data = _norm_negation.sub(template.negation, data) + # Normalize double negation (convert "-(-x)" to x) + data = _norm_double_negation.sub('', data) # Remove multiplication by 1 data = _norm_timesone.sub('', data) # Normalize consecutive whitespace to a single space @@ -135,3 +138,8 @@ def load_and_compare_nl_baseline(baseline, testfile, version='nl'): return nl_diff( *load_nl_baseline(baseline, testfile, version), baseline, testfile ) + +if __name__ == '__main__': + import sys + base, test = load_and_compare_nl_baseline(sys.argv[1], sys.argv[2]) + sys.exit(1 if base or test else 0) diff --git a/pyomo/repn/tests/ampl/test_nlv2.py b/pyomo/repn/tests/ampl/test_nlv2.py index f84b830a0f7..d391e4353a6 100644 --- a/pyomo/repn/tests/ampl/test_nlv2.py +++ b/pyomo/repn/tests/ampl/test_nlv2.py @@ -12,14 +12,19 @@ import pyomo.common.unittest as unittest +import io import math +import os import pyomo.repn.plugins.nl_writer as nl_writer from pyomo.common.log import LoggingIntercept +from pyomo.common.tempfiles import TempfileManager from pyomo.core.expr.current import Expr_if, inequality from pyomo.core.base.expression import ScalarExpression -from pyomo.environ import ConcreteModel, Param, Var, log +from pyomo.environ import ( + ConcreteModel, Objective, Param, Var, log, ExternalFunction, +) class INFO(object): @@ -655,3 +660,53 @@ class CustomExpression(ScalarExpression): self.assertEqual(repn.linear, [(id(m.x), 1)]) self.assertEqual(repn.nonlinear, None) self.assertEqual(info, [None, None, False]) + +class Test_NLWriter(unittest.TestCase): + def test_external_function_str_args(self): + m = ConcreteModel() + m.x = Var() + m.e = ExternalFunction(library='tmp', function='test') + m.o = Objective(expr=m.e(m.x, 'str')) + + # Test explicit newline translation + OUT = io.StringIO(newline='\r\n') + with LoggingIntercept() as LOG: + nl_writer.NLWriter().write(m, OUT) + self.assertIn( + "Writing NL file containing string arguments to a " + "text output stream with line endings other than '\\n' ", + LOG.getvalue() + ) + + # Test system-dependent newline translation + with TempfileManager: + fname = TempfileManager.create_tempfile() + with open(fname, 'w') as OUT: + with LoggingIntercept() as LOG: + nl_writer.NLWriter().write(m, OUT) + if os.linesep == '\n': + self.assertEqual(LOG.getvalue(), "") + else: + self.assertIn( + "Writing NL file containing string arguments to a " + "text output stream with line endings other than '\\n' ", + LOG.getvalue() + ) + + # Test objects lacking 'tell': + r,w = os.pipe() + try: + OUT = os.fdopen(w, 'w') + with LoggingIntercept() as LOG: + nl_writer.NLWriter().write(m, OUT) + if os.linesep == '\n': + self.assertEqual(LOG.getvalue(), "") + else: + self.assertIn( + "Writing NL file containing string arguments to a " + "text output stream that does not support tell()", + LOG.getvalue() + ) + finally: + OUT.close() + os.close(r) diff --git a/pyomo/repn/tests/ampl/test_suffixes.py b/pyomo/repn/tests/ampl/test_suffixes.py index fd0d3f5b33a..269d656b249 100644 --- a/pyomo/repn/tests/ampl/test_suffixes.py +++ b/pyomo/repn/tests/ampl/test_suffixes.py @@ -28,7 +28,7 @@ currdir = this_file_dir() -class TestSuffix(unittest.TestCase): +class SuffixTester(object): @classmethod def setUpClass(cls): cls.context = TempfileManager.new_context() @@ -74,7 +74,7 @@ def test_EXPORT_suffixes_int(self): _test = os.path.join(self.tempdir, "EXPORT_suffixes.test.nl") model.write(filename=_test, - format=ProblemFormat.nl, + format=self.nl_version, io_options={"symbolic_solver_labels": False}) _base = os.path.join(currdir, "EXPORT_suffixes_int.baseline.nl") self.assertEqual(*load_and_compare_nl_baseline(_base, _test)) @@ -115,7 +115,7 @@ def test_EXPORT_suffixes_float(self): _test = os.path.join(self.tempdir, "EXPORT_suffixes.test.nl") model.write(filename=_test, - format=ProblemFormat.nl, + format=self.nl_version, io_options={"symbolic_solver_labels" : False}) _base = os.path.join(currdir, "EXPORT_suffixes_float.baseline.nl") self.assertEqual(*load_and_compare_nl_baseline(_base, _test)) @@ -140,7 +140,7 @@ def test_EXPORT_suffixes_with_SOSConstraint_duplicateref(self): RuntimeError, "NL file writer does not allow both manually " "declared 'ref' suffixes as well as SOSConstraint "): model.write(filename=os.path.join(self.tempdir, "junk.nl"), - format=ProblemFormat.nl, + format=self.nl_version, io_options={"symbolic_solver_labels" : False}) # Test that user defined sosno suffixes fail to @@ -163,7 +163,7 @@ def test_EXPORT_suffixes_with_SOSConstraint_duplicatesosno(self): RuntimeError, "NL file writer does not allow both manually " "declared 'sosno' suffixes as well as SOSConstraint "): model.write(filename=os.path.join(self.tempdir, "junk.nl"), - format=ProblemFormat.nl, + format=self.nl_version, io_options={"symbolic_solver_labels" : False}) # Test that user defined sosno suffixes fail to @@ -186,8 +186,15 @@ def test_EXPORT_suffixes_no_datatype(self): RuntimeError, "NL file writer does not allow both manually " "declared 'sosno' suffixes as well as SOSConstraint "): model.write(filename=os.path.join(self.tempdir, "junk.nl"), - format=ProblemFormat.nl, + format=self.nl_version, io_options={"symbolic_solver_labels" : False}) + +class TestSuffix_nlv1(SuffixTester, unittest.TestCase): + nl_version = 'nl_v1' + +class TestSuffix_nlv2(SuffixTester, unittest.TestCase): + nl_version = 'nl_v2' + if __name__ == "__main__": unittest.main() diff --git a/pyomo/scripting/solve_config.py b/pyomo/scripting/solve_config.py index 174896c6897..fcedb844d6e 100644 --- a/pyomo/scripting/solve_config.py +++ b/pyomo/scripting/solve_config.py @@ -107,14 +107,12 @@ def minlp_config_block(init=False): None, )).declare_as_argument(dest='symbolic_solver_labels') model.declare('file determinism', ConfigValue( - 1, + None, int, 'When interfacing with a solver using file based I/O, set ' 'the effort level for ensuring the file creation process is ' - 'determistic. The default (1) sorts the index of components ' - 'when transforming the model. Anything less than 1 disables ' - 'index sorting. Anything greater than 1 additionally sorts ' - 'by component name to override declaration order.', + 'determistic. See the individual solver interfaces for ' + 'valid values and default level of file determinism.', None, )).declare_as_argument(dest='file_determinism') diff --git a/pyomo/scripting/util.py b/pyomo/scripting/util.py index bd618eae01f..8bb4b701c1b 100644 --- a/pyomo/scripting/util.py +++ b/pyomo/scripting/util.py @@ -444,7 +444,7 @@ def create_model(data): io_options = {} if data.options.model.symbolic_solver_labels: io_options['symbolic_solver_labels'] = True - if data.options.model.file_determinism != 1: + if data.options.model.file_determinism is not None: io_options['file_determinism'] = data.options.model.file_determinism (fname, smap_id) = instance.write(filename=fname, format=format, @@ -550,7 +550,7 @@ def apply_optimizer(data, instance=None): keywords['keepfiles'] = True if data.options.model.symbolic_solver_labels: keywords['symbolic_solver_labels'] = True - if data.options.model.file_determinism != 1: + if data.options.model.file_determinism is not None: keywords['file_determinism'] = data.options.model.file_determinism keywords['tee'] = data.options.runtime.stream_output keywords['timelimit'] = getattr(data.options.solvers[0].options, 'timelimit', 0) diff --git a/pyomo/solvers/tests/piecewise_linear/test_examples.py b/pyomo/solvers/tests/piecewise_linear/test_examples.py index 69ed1975f2e..3b7364231ca 100644 --- a/pyomo/solvers/tests/piecewise_linear/test_examples.py +++ b/pyomo/solvers/tests/piecewise_linear/test_examples.py @@ -33,8 +33,9 @@ class Test(unittest.TestCase): def run_convert2nl(self, name): os.chdir(currdir) - return convert.pyomo2nl(['--symbolic-solver-labels' - ,join(scriptdir,name)]) + return convert.pyomo2nl(['--symbolic-solver-labels', + '--file-determinism', '1', + join(scriptdir,name)]) def run_convert2lp(self, name): os.chdir(currdir) diff --git a/pyomo/solvers/tests/testcases.py b/pyomo/solvers/tests/testcases.py index d15562a959f..96dc6c3a143 100644 --- a/pyomo/solvers/tests/testcases.py +++ b/pyomo/solvers/tests/testcases.py @@ -85,10 +85,15 @@ "Cplex does not report duals of quadratic constraints.") MissingSuffixFailures['cplex', 'nl', 'QCP_simple'] = ( - lambda v: v <= (12,5,9,9), + lambda v: v < (12,6,0,0), {'dual': (True, {'qc0','qc1'})}, "Cplex does not report duals of quadratic constraints.") +SkipTests['cplex', 'nl', 'QCP_simple'] = ( + lambda v: v == (12,6,0,0), + "Cplex 12.6.0.0 produces inconsistent dual values based on " + "NL variable ordering (which changes between the NLv1 and NLv2 writers") + # # GUROBI #