repopackpy-output.txt

================================================================
RepopackPy Output File
================================================================

This file was generated by RepopackPy on: 2024-11-18T14:32:21.374737

Purpose:
--------
This file contains a packed representation of the entire repository's contents.
It is designed to be easily consumable by AI systems for analysis, code review,
or other automated processes.

File Format:
------------
The content is organized as follows:
1. This header section
2. Repository structure
3. Multiple file entries, each consisting of:
  a. A separator line (================)
  b. The file path (File: path/to/file)
  c. Another separator line
  d. The full contents of the file
  e. A blank line

Usage Guidelines:
-----------------
1. This file should be treated as read-only. Any changes should be made to the
  original repository files, not this packed version.
2. When processing this file, use the separators and "File:" markers to
  distinguish between different files in the repository.
3. Be aware that this file may contain sensitive information. Handle it with
  the same level of security as you would the original repository.

Notes:
------
- Some files may have been excluded based on .gitignore rules and RepopackPy's
  configuration.
- Binary files are not included in this packed representation. Please refer to
  the Repository Structure section for a complete list of file paths, including
  binary files.

For more information about RepopackPy, visit: https://github.com/abinthomasonline/repopack-py

================================================================
Repository Structure
================================================================
.github\workflows\test.yml
.readthedocs.yml
LICENSE
MANIFEST.in
README.md
build_test_data.py
docs\Makefile
docs\make.bat
docs\requirements.txt
docs\source\Spot.html
docs\source\conf.py
docs\source\examples.rst
docs\source\historical.rst
docs\source\index.rst
docs\source\installation.rst
docs\source\intro.rst
docs\source\markets.rst
docs\source\publications.rst
docs\source\time_sequential.rst
nempy\__init__.py
nempy\help_functions\__init__.py
nempy\help_functions\helper_functions.py
nempy\markets.py
nempy\spot_markert_backend\__init__.py
nempy\spot_markert_backend\check.py
nempy\spot_markert_backend\dataframe_validator.py
nempy\spot_markert_backend\elastic_constraints.py
nempy\spot_markert_backend\fcas_constraints.py
nempy\spot_markert_backend\interconnectors.py
nempy\spot_markert_backend\market_constraints.py
nempy\spot_markert_backend\objective_function.py
nempy\spot_markert_backend\solver_interface.py
nempy\spot_markert_backend\unit_constraints.py
nempy\spot_markert_backend\variable_ids.py
nempy\time_sequential.py
publications\all_features_example.py
publications\energy_only_market.py
pyproject.toml
pytest.ini
tests\build_historical_test_data_cache_and_db.py
tests\get_violation_times.py
tests\historical_market_builder.py
tests\test_constraint_equation_calc.py
tests\test_historical.py
tests\test_historical_spot_market_inputs.py
tests\test_interconnector_loss_functions.py
tests\test_market_constraints.py
tests\test_markets.py
tests\test_objective_function.py
tests\test_rpn_calc.py
tests\test_solver_interface.py
tests\test_unit_constraints.py
tests\test_variable_ids.py

================================================================
Repository Files
================================================================

================
File: .readthedocs.yml
================
# .readthedocs.yaml
# Read the Docs configuration file
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details

# Required
version: 2

# Set the version of Python and other tools you might need
build:
  os: ubuntu-20.04
  tools:
    python: "3.9"
  jobs:
    post_install:
      - pip install poetry
      - poetry config virtualenvs.create false
      - poetry install --with=docs

# Build documentation in the docs/ directory with Sphinx
sphinx:
   configuration: docs/source/conf.py

# If using Sphinx, optionally build your docs in additional formats such as PDFf
formats:
   - pdf

================
File: build_test_data.py
================
import sqlite3
from nempy.historical_inputs import mms_db, xml_cache


con = sqlite3.connect('market_management_system.db')
mms_db_manager = mms_db.DBManager(connection=con)
xml_cache_manager = xml_cache.XMLCacheManager('test_nemde_cache')
mms_db_manager.populate(start_year=2019, start_month=1, end_year=2019, end_month=1)
mms_db_manager._create_sample_database('2019/01/10 12:05:00')
xml_cache_manager.populate(start_year=2019, start_month=1, end_year=2019, end_month=1)

================
File: LICENSE
================
BSD 3-Clause License

Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team
All rights reserved.

Copyright (c) 2011-2020, Open source contributors.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of the copyright holder nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

================
File: MANIFEST.in
================
include *.txt
include LICENSE
include pytest.ini
recursive-include docs *.bat
recursive-include docs *.html
recursive-include docs *.pdf
recursive-include docs *.py
recursive-include docs *.rst
recursive-include docs *.txt
recursive-include docs Makefile
recursive-include examples *.py
recursive-include nempy *.py
recursive-include publications *.py
recursive-include tests *.csv
recursive-include tests *.py
recursive-include tests *.zip
recursive-exclude tests/test_files/historical_xml_files *.loaded

================
File: pyproject.toml
================
[project]
name = "nempy"
version = "2.2.0"
description="A flexible tool kit for modelling Australia's National Electricity Market dispatch procedure."
authors = [
    { name = "nick-gorman", email = "n.gorman305@gmail.com" },
]
dependencies = [
    "mip==1.16rc0",
    "pandas>=2.2.2",
    "xmltodict==0.12.0",
    "requests>=2.0.0",
    "repopack>=0.1.4",
]
readme = "README.md"
requires-python = ">= 3.9"

[build-system]
requires = ["setuptools>=61.0", "wheel"]
build-backend = "setuptools.build_meta"

[tool.uv]
managed = true
dev-dependencies = [
    "Sphinx>=5.0.0",
    "autodocsumm>=0.2.11",
    "sphinx-rtd-theme>=1.3.0",
    "pytest>=8.3.2",
]

[tool.pytest.ini_options]
# path to tests for pytest
testpaths = ["nempy", "tests"]
# addopts = add options
addopts = "-ra --doctest-modules"

[tool.setuptools.packages.find]
where = ["nempy"]

================
File: pytest.ini
================
[pytest]
addopts = --doctest-modules --ignore=tests/test_historical.py --ignore=tests/get_violation_times.py --ignore=tests/build_historical_test_data_cache_and_db.py --ignore=tests/test_files --ignore=nempy/historical_inputs/test_nemde_cache
testpaths = tests nempy

================
File: README.md
================
# Nempy

[![Current build](https://github.com/UNSW-CEEM/nempy/actions/workflows/test.yml/badge.svg)](https://github.com/UNSW-CEEM/nempy/actions/workflows/test.yml)
[![Documentation](https://readthedocs.org/projects/nempy/badge/?version=latest)](https://nempy.readthedocs.io/en/latest/?badge=latest)
[![DOI](https://joss.theoj.org/papers/10.21105/joss.03596/status.svg)](https://doi.org/10.21105/joss.03596)

## Table of Contents
- [Introduction](https://github.com/UNSW-CEEM/nempy#introduction)
- [Installation](https://github.com/UNSW-CEEM/nempy#installation)
- [Documentation](https://github.com/UNSW-CEEM/nempy#documentation)
- [Community](https://github.com/UNSW-CEEM/nempy#community)
- [Author](https://github.com/UNSW-CEEM/nempy#author)
- [Citation](https://github.com/UNSW-CEEM/nempy#citation)
- [License](https://github.com/UNSW-CEEM/nempy#license)
- [Examples](https://github.com/UNSW-CEEM/nempy#examples)

## Introduction

Nempy is a Python package for modelling the dispatch procedure of the Australian National Electricity Market (NEM). The idea is 
that you can start simple and grow the complexity of your model by adding features such as 
ramping constraints, interconnectors, FCAS markets and more. See the [examples](https://github.com/UNSW-CEEM/nempy#examples) below.

| ![nempy-accuracy](https://github.com/prakaa/nempy/assets/40549624/6a994cee-3255-4e3d-b04b-6d4d7e155065) | 
|:--:| 
| *Dispatch price results from the New South Wales region for 1000 randomly selected intervals in the 2019 calendar year. The actual prices, prior to scaling or capping, are also shown for comparison. Results from two Nempy models are shown, one with a full set of dispatch features, and one without FCAS markets or generic constraints (network and security constraints). Actual prices, results from the full featured model, and the simpler model are shown in descending order for actual prices, results from the simpler model are also shown resorted.* |

For further details, refer to the [documentation](https://nempy.readthedocs.io/en/latest/intro.html#).

For a brief introduction to the NEM, refer to this [ document](https://aemo.com.au/-/media/Files/Electricity/NEM/National-Electricity-Market-Fact-Sheet.pdf).

## Installation
Installing Nempy to use in your project is easy.

```bash
pip install nempy
```

## Documentation

A more detailed introduction to Nempy, examples, and reference documentation can be found on the 
[readthedocs](https://nempy.readthedocs.io/en/latest/) page.

## Community

Nempy is open-source and we welcome all forms of community engagement.

### Support

You can seek support for using Nempy using the [discussion tab on GitHub](https://github.com/UNSW-CEEM/nempy/discussions), checking the [issues register](https://github.com/UNSW-CEEM/nempy/issues), or by contacting Nick directly (n.gorman at unsw.edu.au).

If you cannot find a pre-existing issue related to your enquiry, you can submit a new one via the [issues register](https://github.com/UNSW-CEEM/nempy/issues). Issue submissions do not need to adhere to any particular format.

### Future support and maintenance

Planning to continue support and maintenance for Nempy after the PhD project is complete is currently underway. If Nempy
is useful to your work, research, or business, please reach out and inform us so we can consider your use case and
needs.

### Contributing

Contributions via pull requests are welcome. Contributions should:

1. Follow the PEP8 style guide (with exception of line length up to 120 rather than 80)
2. Ensure that all existing automated tests continue to pass (unless you are explicitly changing intended behavour; if you are, please highlight this in your pull request description)
3. Implement automated tests for new features
4. Provide doc strings for public interfaces

#### Installation for development

Nempy uses [`poetry`](https://python-poetry.org/docs/) as a dependency and project management tool. To install Nempy for development, clone or fork the repo and then run the following command in the main directory to install required dependencies and the source code as an editable project:

```bash
poetry install --with=dev
```
You can then work within the virtual environment using `poetry shell` or run commands within it using `poetry run`.

## Author

Nempy's development is being led by Nick Gorman as part of his PhD candidature at the Collaboration on Energy and Environmental
Markets at the University of New South Wales' School of Photovoltaics and Renewable Energy Engineering. (https://www.ceem.unsw.edu.au/). 

## Citation

If you use Nempy, please cite the package via the [JOSS paper](https://doi.org/10.5281/zenodo.7397514) (suggested citation below):
> Gorman et al., (2022). Nempy: A Python package for modelling the Australian National Electricity Market dispatch procedure. Journal of Open Source Software, 7(70), 3596, https://doi.org/10.21105/joss.03596

## License

Nempy was created by Nicholas Gorman. It is licensed under the terms of [the BSD 3-Clause Licence](./LICENSE).

## Examples
<details>

<summary>A simple example</summary>

```python
import pandas as pd
from nempy import markets

# Volume of each bid, number of bands must equal number of bands in price_bids.
volume_bids = pd.DataFrame({
    'unit': ['A', 'B'],
    '1': [20.0, 50.0],  # MW
    '2': [20.0, 30.0],  # MW
    '3': [5.0, 10.0]  # More bid bands could be added.
})

# Price of each bid, bids must be monotonically increasing.
price_bids = pd.DataFrame({
    'unit': ['A', 'B'],
    '1': [50.0, 50.0],  # $/MW
    '2': [60.0, 55.0],  # $/MW
    '3': [100.0, 80.0]  # . . .
})

# Other unit properties
unit_info = pd.DataFrame({
    'unit': ['A', 'B'],
    'region': ['NSW', 'NSW'],  # MW
})

# The demand in the region\s being dispatched
demand = pd.DataFrame({
    'region': ['NSW'],
    'demand': [120.0]  # MW
})

# Create the market model
market = markets.SpotMarket(unit_info=unit_info, 
                            market_regions=['NSW'])
market.set_unit_volume_bids(volume_bids)
market.set_unit_price_bids(price_bids)
market.set_demand_constraints(demand)

# Calculate dispatch and pricing
market.dispatch()

# Return the total dispatch of each unit in MW.
print(market.get_unit_dispatch())
#   unit service  dispatch
# 0    A  energy      40.0
# 1    B  energy      80.0

# Return the price of energy in each region.
print(market.get_energy_prices())
#   region  price
# 0    NSW   60.0
```

</details>

<details>

<summary>A detailed example</summary>

The example demonstrates the broad range of market features that can be implemented with Nempy and the use of auxiliary 
modelling tools for accessing historical market data published by AEMO and preprocessing it for compatibility with Nempy.

> [!WARNING]  
> This example downloads approximately 54 GB of data from AEMO.

```python
# Notice:
# - This script downloads large volumes of historical market data (~54 GB) from AEMO's nemweb
#   portal. You can also reduce the data usage by restricting the time window given to the
#   xml_cache_manager and in the get_test_intervals function. The boolean on line 23 can
#   also be changed to prevent this happening repeatedly once the data has been downloaded.

import sqlite3
from datetime import datetime, timedelta
import random
import pandas as pd
from nempy import markets
from nempy.historical_inputs import loaders, mms_db, \
    xml_cache, units, demand, interconnectors, constraints, rhs_calculator
from nempy.help_functions.helper_functions import update_rhs_values

con = sqlite3.connect('D:/nempy_2021/historical_mms.db')
mms_db_manager = mms_db.DBManager(connection=con)

xml_cache_manager = xml_cache.XMLCacheManager('D:/nempy_2021/xml_cache')

# The second time this example is run on a machine this flag can
# be set to false to save downloading the data again.
download_inputs = True

if download_inputs:
    # This requires approximately 4 GB of storage.
    mms_db_manager.populate(start_year=2021, start_month=12,
                            end_year=2022, end_month=1)

    # This requires approximately 50 GB of storage.
    xml_cache_manager.populate_by_day(start_year=2021, start_month=12, start_day=1,
                                      end_year=2021, end_month=12, end_day=31)

raw_inputs_loader = loaders.RawInputsLoader(
    nemde_xml_cache_manager=xml_cache_manager,
    market_management_system_database=mms_db_manager)


# A list of intervals we want to recreate historical dispatch for.
def get_test_intervals(number=100):
    start_time = datetime(year=2021, month=12, day=1, hour=0, minute=0)
    end_time = datetime(year=2021, month=12, day=31, hour=0, minute=0)
    difference = end_time - start_time
    difference_in_5_min_intervals = difference.days * 12 * 24
    random.seed(1)
    intervals = random.sample(range(1, difference_in_5_min_intervals), number)
    times = [start_time + timedelta(minutes=5 * i) for i in intervals]
    times_formatted = [t.isoformat().replace('T', ' ').replace('-', '/') for t in times]
    return times_formatted


# List for saving outputs to.
outputs = []
c = 0
# Create and dispatch the spot market for each dispatch interval.
for interval in get_test_intervals(number=100):
    c += 1
    print(str(c) + ' ' + str(interval))
    raw_inputs_loader.set_interval(interval)
    unit_inputs = units.UnitData(raw_inputs_loader)
    interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
    constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
    demand_inputs = demand.DemandData(raw_inputs_loader)
    rhs_calculation_engine = rhs_calculator.RHSCalc(xml_cache_manager)

    unit_info = unit_inputs.get_unit_info()
    market = markets.SpotMarket(market_regions=['QLD1', 'NSW1', 'VIC1',
                                                'SA1', 'TAS1'],
                                unit_info=unit_info)

    # Set bids
    volume_bids, price_bids = unit_inputs.get_processed_bids()
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_price_bids(price_bids)

    # Set bid in capacity limits
    unit_bid_limit = unit_inputs.get_unit_bid_availability()
    market.set_unit_bid_capacity_constraints(unit_bid_limit)
    cost = constraint_inputs.get_constraint_violation_prices()['unit_capacity']
    market.make_constraints_elastic('unit_bid_capacity', violation_cost=cost)

    # Set limits provided by the unconstrained intermittent generation
    # forecasts. Primarily for wind and solar.
    unit_uigf_limit = unit_inputs.get_unit_uigf_limits()
    market.set_unconstrained_intermitent_generation_forecast_constraint(
        unit_uigf_limit)
    cost = constraint_inputs.get_constraint_violation_prices()['uigf']
    market.make_constraints_elastic('uigf_capacity', violation_cost=cost)

    # Set unit ramp rates.
    def set_ramp_rates(run_type):
        ramp_rates = unit_inputs.get_ramp_rates_used_for_energy_dispatch(run_type=run_type)
        market.set_unit_ramp_up_constraints(
            ramp_rates.loc[:, ['unit', 'initial_output', 'ramp_up_rate']])
        market.set_unit_ramp_down_constraints(
            ramp_rates.loc[:, ['unit', 'initial_output', 'ramp_down_rate']])
        cost = constraint_inputs.get_constraint_violation_prices()['ramp_rate']
        market.make_constraints_elastic('ramp_up', violation_cost=cost)
        market.make_constraints_elastic('ramp_down', violation_cost=cost)


    set_ramp_rates(run_type='fast_start_first_run')

    # Set unit FCAS trapezium constraints.
    unit_inputs.add_fcas_trapezium_constraints()
    cost = constraint_inputs.get_constraint_violation_prices()['fcas_max_avail']
    fcas_availability = unit_inputs.get_fcas_max_availability()
    market.set_fcas_max_availability(fcas_availability)
    market.make_constraints_elastic('fcas_max_availability', cost)
    cost = constraint_inputs.get_constraint_violation_prices()['fcas_profile']
    regulation_trapeziums = unit_inputs.get_fcas_regulation_trapeziums()
    market.set_energy_and_regulation_capacity_constraints(regulation_trapeziums)
    market.make_constraints_elastic('energy_and_regulation_capacity', cost)
    contingency_trapeziums = unit_inputs.get_contingency_services()
    market.set_joint_capacity_constraints(contingency_trapeziums)
    market.make_constraints_elastic('joint_capacity', cost)


    def set_joint_ramping_constraints(run_type):
        cost = constraint_inputs.get_constraint_violation_prices()['fcas_profile']
        scada_ramp_down_rates = unit_inputs.get_scada_ramp_down_rates_of_lower_reg_units(
            run_type=run_type)
        market.set_joint_ramping_constraints_lower_reg(scada_ramp_down_rates)
        market.make_constraints_elastic('joint_ramping_lower_reg', cost)
        scada_ramp_up_rates = unit_inputs.get_scada_ramp_up_rates_of_raise_reg_units(
            run_type=run_type)
        market.set_joint_ramping_constraints_raise_reg(scada_ramp_up_rates)
        market.make_constraints_elastic('joint_ramping_raise_reg', cost)


    set_joint_ramping_constraints(run_type="fast_start_first_run")

    # Set interconnector definitions, limits and loss models.
    interconnectors_definitions = \
        interconnector_inputs.get_interconnector_definitions()
    loss_functions, interpolation_break_points = \
        interconnector_inputs.get_interconnector_loss_model()
    market.set_interconnectors(interconnectors_definitions)
    market.set_interconnector_losses(loss_functions,
                                     interpolation_break_points)

    # Calculate rhs constraint values that depend on the basslink frequency controller from scratch so there is
    # consistency between the basslink switch runs.
    # Find the constraints that need to be calculated because they depend on the frequency controller status.
    constraints_to_update = (
        rhs_calculation_engine.get_rhs_constraint_equations_that_depend_value('BL_FREQ_ONSTATUS', 'W'))
    initial_bl_freq_onstatus = rhs_calculation_engine.scada_data['W']['BL_FREQ_ONSTATUS'][0]['@Value']
    # Calculate new rhs values for the constraints that need updating.
    new_rhs_values = rhs_calculation_engine.compute_constraint_rhs(constraints_to_update)

    # Add generic constraints and FCAS market constraints.
    fcas_requirements = constraint_inputs.get_fcas_requirements()
    fcas_requirements = update_rhs_values(fcas_requirements, new_rhs_values)
    market.set_fcas_requirements_constraints(fcas_requirements)
    violation_costs = constraint_inputs.get_violation_costs()
    market.make_constraints_elastic('fcas', violation_cost=violation_costs)
    generic_rhs = constraint_inputs.get_rhs_and_type_excluding_regional_fcas_constraints()
    generic_rhs = update_rhs_values(generic_rhs, new_rhs_values)
    market.set_generic_constraints(generic_rhs)
    market.make_constraints_elastic('generic', violation_cost=violation_costs)

    unit_generic_lhs = constraint_inputs.get_unit_lhs()
    market.link_units_to_generic_constraints(unit_generic_lhs)
    interconnector_generic_lhs = constraint_inputs.get_interconnector_lhs()
    market.link_interconnectors_to_generic_constraints(
        interconnector_generic_lhs)

    # Set the operational demand to be met by dispatch.
    regional_demand = demand_inputs.get_operational_demand()
    market.set_demand_constraints(regional_demand)

    # Set tiebreak constraint to equalise dispatch of equally priced bids.
    cost = constraint_inputs.get_constraint_violation_prices()['tiebreak']
    market.set_tie_break_constraints(cost)

    # Get unit dispatch without fast start constraints and use it to
    # make fast start unit commitment decisions.
    market.dispatch()
    dispatch = market.get_unit_dispatch()
    fast_start_profiles = unit_inputs.get_fast_start_profiles_for_dispatch(dispatch)
    set_ramp_rates(run_type='fast_start_second_run')
    set_joint_ramping_constraints(run_type='fast_start_second_run')
    market.set_fast_start_constraints(fast_start_profiles)
    if 'fast_start' in market.get_constraint_set_names.keys():
        cost = constraint_inputs.get_constraint_violation_prices()['fast_start']
        market.make_constraints_elastic('fast_start', violation_cost=cost)

    # First run of Basslink switch runs
    market.dispatch()  # First dispatch without allowing over constrained dispatch re-run to get objective function.
    objective_value_run_one = market.objective_value
    if constraint_inputs.is_over_constrained_dispatch_rerun():
        market.dispatch(allow_over_constrained_dispatch_re_run=True,
                        energy_market_floor_price=-1000.0,
                        energy_market_ceiling_price=15000.0,
                        fcas_market_ceiling_price=1000.0)
    prices_run_one = market.get_energy_prices()  # If this is the lowest cost run these will be the market prices.

    # Re-run dispatch with Basslink Frequency controller off.
    # Set frequency controller to off in rhs calculations
    rhs_calculation_engine.update_spd_id_value('BL_FREQ_ONSTATUS', 'W', '0')
    new_bl_freq_onstatus = rhs_calculation_engine.scada_data['W']['BL_FREQ_ONSTATUS'][0]['@Value']
    # Find the constraints that need to be updated because they depend on the frequency controller status.
    constraints_to_update = (
        rhs_calculation_engine.get_rhs_constraint_equations_that_depend_value('BL_FREQ_ONSTATUS', 'W'))
    # Calculate new rhs values for the constraints that need updating.
    new_rhs_values = rhs_calculation_engine.compute_constraint_rhs(constraints_to_update)
    # Update the constraints in the market.
    fcas_requirements = update_rhs_values(fcas_requirements, new_rhs_values)
    violation_costs = constraint_inputs.get_violation_costs()
    market.set_fcas_requirements_constraints(fcas_requirements)
    market.make_constraints_elastic('fcas', violation_cost=violation_costs)
    generic_rhs = update_rhs_values(generic_rhs, new_rhs_values)
    market.set_generic_constraints(generic_rhs)
    market.make_constraints_elastic('generic', violation_cost=violation_costs)

    # Reset ramp rate constraints for first run of second Basslink switchrun
    set_ramp_rates(run_type='fast_start_first_run')
    set_joint_ramping_constraints(run_type='fast_start_first_run')

    # Get unit dispatch without fast start constraints and use it to
    # make fast start unit commitment decisions.
    market.remove_fast_start_constraints()
    market.dispatch()
    dispatch = market.get_unit_dispatch()
    fast_start_profiles = unit_inputs.get_fast_start_profiles_for_dispatch(dispatch)
    set_ramp_rates(run_type='fast_start_second_run')
    set_joint_ramping_constraints(run_type='fast_start_second_run')
    market.set_fast_start_constraints(fast_start_profiles)
    if 'fast_start' in market.get_constraint_set_names():
        cost = constraint_inputs.get_constraint_violation_prices()['fast_start']
        market.make_constraints_elastic('fast_start', violation_cost=cost)

    market.dispatch()  # First dispatch without allowing over constrained dispatch re-run to get objective function.
    objective_value_run_two = market.objective_value
    if constraint_inputs.is_over_constrained_dispatch_rerun():
        market.dispatch(allow_over_constrained_dispatch_re_run=True,
                        energy_market_floor_price=-1000.0,
                        energy_market_ceiling_price=15000.0,
                        fcas_market_ceiling_price=1000.0)
    prices_run_two = market.get_energy_prices()  # If this is the lowest cost run these will be the market prices.

    prices_run_one['time'] = interval
    prices_run_two['time'] = interval

    # Getting historical prices for comparison. Note, ROP price, which is
    # the regional reference node price before the application of any
    # price scaling by AEMO, is used for comparison.
    historical_prices = mms_db_manager.DISPATCHPRICE.get_data(interval)

    # The prices from the run with the lowest objective function value are used.
    if objective_value_run_one < objective_value_run_two:
        prices = prices_run_one
    else:
        prices = prices_run_two

    prices['time'] = interval
    prices = pd.merge(prices, historical_prices,
                      left_on=['time', 'region'],
                      right_on=['SETTLEMENTDATE', 'REGIONID'])

    outputs.append(prices)

con.close()

outputs = pd.concat(outputs)

outputs['error'] = outputs['price'] - outputs['ROP']

print('\n Summary of error in energy price volume weighted average price. \n'
      'Comparison is against ROP, the price prior to \n'
      'any post dispatch adjustments, scaling, capping etc.')
print('Mean price error: {}'.format(outputs['error'].mean()))
print('Median price error: {}'.format(outputs['error'].quantile(0.5)))
print('5% percentile price error: {}'.format(outputs['error'].quantile(0.05)))
print('95% percentile price error: {}'.format(outputs['error'].quantile(0.95)))

#  Summary of error in energy price volume weighted average price.
# Comparison is against ROP, the price prior to
# any post dispatch adjustments, scaling, capping etc.
# Mean price error: -0.3284696359015098
# Median price error: 0.0
# 5% percentile price error: -0.5389930178124978
# 95% percentile price error: 0.13746097842649457
```
</details>

================
File: .github\workflows\test.yml
================
name: Run Nempy Tests
on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master
  release:
    types: [ created ]
  workflow_dispatch: # manual trigger
  
jobs:
  test:
    # Matrix testing to test across OSs and Python versions
    # Fail-fast: fail the entire job as soon as anything fails
    strategy:
      fail-fast: true
      matrix:
        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
      # Necessary for poetry & Windows
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Get poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: ~/.cache
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
      
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install poetry env with dev group
        run: poetry install --with=dev

      - name: Run tests with pytest
        run: |
          source $VENV
          python -m pytest
          
  publish:
    if: github.event_name == 'release' && github.event.action == 'created'
    needs: test
    runs-on: ubuntu-latest
    steps:
      # Checkout the repo so the workflow can access it
      - name: Checkout
        uses: actions/checkout@v3

      # Set up Python
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'

      # Install the specified version of poetry
      - name: Get poetry
        uses: snok/install-poetry@v1
        with:
          version: 1.4.0
          virtualenvs-create: true
          virtualenvs-in-project: true

      # Load cached poetry env if it exists
      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v2
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      # Build and publish to PyPI
      - name: Build and publish # publish tsgen to PyPI
        env:
          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: poetry publish -u $PYPI_USERNAME -p $PYPI_PASSWORD --build

================
File: docs\make.bat
================
@ECHO OFF

pushd %~dp0

REM Command file for Sphinx documentation

if "%SPHINXBUILD%" == "" (
	set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=source
set BUILDDIR=build

if "%1" == "" goto help

%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
	echo.
	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
	echo.installed, then set the SPHINXBUILD environment variable to point
	echo.to the full path of the 'sphinx-build' executable. Alternatively you
	echo.may add the Sphinx directory to PATH.
	echo.
	echo.If you don't have Sphinx installed, grab it from
	echo.http://sphinx-doc.org/
	exit /b 1
)

%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
goto end

:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%

:end
popd

================
File: docs\Makefile
================
# Minimal makefile for Sphinx documentation
#

# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS    ?=
SPHINXBUILD   ?= sphinx-build
SOURCEDIR     = source
BUILDDIR      = build

# Put it first so that "make" without argument is like "make help".
help:
	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

.PHONY: help Makefile

# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

================
File: docs\requirements.txt
================
autodocsumm==0.1.13

================
File: docs\source\conf.py
================
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('../../'))

# -- Project information -----------------------------------------------------

project = 'nempy'
copyright = '2020, Nick Gorman'
author = 'Nick Gorman'

# The full version, including alpha/beta/rc tags
release = '0.0.1'


# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx_rtd_theme', 'sphinx.ext.autosectionlabel', "sphinx.ext.autodoc", 'sphinx.ext.napoleon',
              'sphinx.ext.autosummary', 'autodocsumm']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = []


# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

master_doc = 'index'

autodoc_mock_imports = ["pandas", "numpy", "mip", "xmltodict"]

autodoc_member_order = 'bysource'

napoleon_custom_sections = ['Multiple Returns']

================
File: docs\source\examples.rst
================
.. _examples1:

Examples
====================
A number of examples of how to use Nempy are provided below. Examples 1 to 5 are simple and aim introduce various
market features that can be modelled with Nempy in an easy to understand way, the dispatch and pricing outcomes are
explained in inline comments where the results are printed. Examples 6 and 7 show how to use the historical data input
preparation tools provided with Nempy to recreate historical dispatch intervals. Historical dispatch and pricing
outcomes can be difficult to interpret as they are usually the result of complex interactions between the many features
of the dispatch process, for these example the results are plotted in comparison to historical price outcomes.
Example 8 demonstrates how the outputs of one dispatch interval can be used as the initial conditions of the
next dispatch interval to create a time sequential model, additionally the current limitations with the approach are
briefly discussed.

1. Bid stack equivalent market
---------------------------
This example implements a one region bid stack model of an electricity market. Under the bid stack model, generators are
dispatched according to their bid prices, from cheapest to most expensive, until all demand is satisfied. No loss factors,
ramping constraints or other factors are considered.

.. literalinclude:: ../../examples/bidstack.py
    :linenos:
    :language: python


2. Unit loss factors, capacities and ramp rates
-----------------------------------------------
A simple example with two units in a one region market, units are given loss factors, capacity values and ramp rates.
The effects of loss factors on dispatch and market prices are explained.

.. literalinclude:: ../../examples/ramp_rates_and_loss_factors.py
    :linenos:
    :language: python


3. Interconnector with losses
-----------------------------
A simple example demonstrating how to implement a two region market with an interconnector. The interconnector is
modelled simply, with a fixed percentage of losses. To make the interconnector flow and loss calculation easy to
understand a single unit is modelled in the NSW region, NSW demand is set zero, and VIC region demand is set to 90 MW,
thus all the power to meet VIC demand must flow across the interconnetcor.

.. literalinclude:: ../../examples/interconnector_constant_loss_percentage.py
    :linenos:
    :language: python


4. Dynamic non-linear interconnector losses
----------------------------------------
This example demonstrates how to model regional demand dependant interconnector loss functions as decribed in the AEMO
:download:`Marginal Loss Factors documentation section 3 to 5  <../../docs/pdfs/Marginal Loss Factors for the 2020-21 Financial year.pdf>`.
To make the interconnector flow and loss calculation easy to understand a single unit is modelled in the NSW region,
NSW demand is set zero, and VIC region demand is set to 800 MW, thus all the power to meet VIC demand must flow across
the interconnetcor.


.. literalinclude:: ../../examples/interconnector_dynamic_losses.py
    :linenos:
    :language: python


5. Simple FCAS markets
----------------------------------------
This example implements a market for energy, regulation raise and contingency 6 sec raise, with
co-optimisation constraints as described in section 6.2 and 6.3 of
:download:`FCAS Model in NEMDE <../../docs/pdfs/FCAS Model in NEMDE.pdf>`.

.. literalinclude:: ../../examples/simple_FCAS_markets.py
    :linenos:
    :language: python


6. Simple recreation of historical dispatch
----------------------------------------
Demonstrates using Nempy to recreate historical dispatch intervals by implementing a simple energy market with unit bids,
unit maximum capacity constraints and interconnector models, all sourced from historical data published by AEMO.

.. image:: ../../examples/charts/energy_market_only_qld_prices.png
  :width: 600

*Results from example: for the QLD region a reasonable fit between modelled prices and historical prices is obtained.*

.. warning:: Warning this script downloads approximately 8.5 GB of data from AEMO. The download_inputs flag can be set
             to false to stop the script re-downloading data for subsequent runs.

.. note:: This example also requires plotly >= 5.3.1, < 6.0.0 and kaleido == 0.2.1. Run pip install plotly==5.3.1 and pip
          install kaleido==0.2.1

.. literalinclude:: ../../examples/recreating_historical_dispatch.py
    :linenos:
    :language: python


7. Detailed recreation of historical dispatch with Basslink switch run
----------------------------------------------------------------------
This example demonstrates using Nempy to recreate historical dispatch intervals by implementing an energy market using
all the features of the Nempy market model, with inputs sourced from historical data published by AEMO. This example has
been updated to include the use of functionality developed to enable modelling the Basslink switch run, which is new in
Nempy version 2.0.0. Previously, Nempy relied on using the generic constraint RHS values reported with the NEMDE
solution from what historically was the least cost case of the switch run. However, the new functionality allows the RHS
values for each case of the switch run to be calculated by Nempy, and so for each case of switch run to be tested.

.. warning:: Warning this script downloads approximately 54 GB of data from AEMO. The download_inputs flag can be set
             to false to stop the script re-downloading data for subsequent runs.

.. literalinclude:: ../../examples/basslink_switchrun.py
    :linenos:
    :language: python


7. Recreation of historical dispatch without Basslink switchrun
------------------------------------------------------------------------
This example demonstrates using Nempy to recreate historical dispatch intervals by implementing an energy market using
all the features of the Nempy market model, except the Basslink switch run, with inputs sourced from historical data
published by AEMO. The main reason not to include Basslink switch run is to speed up runtime. Note each interval is
dispatched as a standalone simulation and the results from one dispatch interval are not carried over to be the initial
conditions of the next interval, rather the historical initial conditions are always used.

.. warning:: Warning this script downloads approximately 54 GB of data from AEMO. The download_inputs flag can be set
             to false to stop the script re-downloading data for subsequent runs.

.. literalinclude:: ../../examples/all_features_example_except_basslink.py
    :linenos:
    :language: python

8. Time sequential recreation of historical dispatch
----------------------------------------------------
This example demonstrates using Nempy to recreate historical dispatch in a dynamic or time sequential manner, this means the outputs
of one interval become the initial conditions for the next dispatch interval. Note, currently there is not the infrastructure
in place to include features such as generic constraints in the time sequential model as the rhs values of many constraints
would need to be re-calculated based on the dynamic system state. Similarly, using historical bids in this example is
some what problematic as participants also dynamically change their bids based on market conditions. However, for the sake
of demonstrating how Nempy can be used to create time sequential models, historical bids are used in this example.

.. warning:: Warning this script downloads approximately 8.5 GB of data from AEMO. The download_inputs flag can be set
             to false to stop the script re-downloading data for subsequent runs.

.. literalinclude:: ../../examples/time_sequential.py
    :linenos:
    :language: python


10. Nempy performance on older data (Jan 2013, without Basslink switch run)
---------------------------------------------------------------------------
This example demonstrates using Nempy to recreate historical dispatch intervals by implementing an energy market using all the
features of the Nempy market model, with inputs sourced from historical data published by AEMO. A set of 100 random dispatch
intervals from January 2015 are dispatched and compared to historical results to see how well Nempy performs for
replicating older versions of the NEM's dispatch procedure. Comparison is against ROP, the region price prior to any post
dispatch adjustments, scaling, capping etc.

Summary of results:

| Mean price error: 0.003
| Median price error: 0.000
| 5% percentile price error: 0.000
| 95% percentile price error: 0.001

.. warning:: Warning this script downloads approximately 54 GB of data from AEMO. The download_inputs flag can be set
             to false to stop the script re-downloading data for subsequent runs.

.. literalinclude:: ../../examples/performance_on_older_data.py
    :linenos:
    :language: python

================
File: docs\source\historical.rst
================
.. _historical:

historical_inputs modules
===============================
The module provides tools for accessing historical market data and preprocessing for compatibility with the SpotMarket
class.

xml_cache
---------------

.. automodule:: nempy.historical_inputs.xml_cache
    :autosummary:
    :members:
    :inherited-members:
    :exclude-members: timedelta,datetime,time,Path,xmltodict

mms_db
-----------------------------------------

.. automodule:: nempy.historical_inputs.mms_db
    :autosummary:
    :members:
    :inherited-members:
    :exclude-members: timedelta,datetime

loaders
-----------------------------------------

.. automodule:: nempy.historical_inputs.loaders
    :autosummary:
    :members:

units
--------

.. automodule:: nempy.historical_inputs.units
    :autosummary:
    :members:

interconnectors
------------------

.. automodule:: nempy.historical_inputs.interconnectors
    :autosummary:
    :members:

demand
------------------

.. automodule:: nempy.historical_inputs.demand
    :autosummary:
    :members:

constraints
------------------

.. automodule:: nempy.historical_inputs.constraints
    :autosummary:
    :members:

RHSCalc
------------------

.. automodule:: nempy.historical_inputs.rhs_calculator
    :autosummary:
    :members:

================
File: docs\source\index.rst
================
.. nempy documentation master file, created by
   sphinx-quickstart on Tue Apr 14 21:20:52 2020.
   You can adapt this file completely to your liking, but it should at least
   contain the root `toctree` directive.

Welcome to nempy's documentation!
=================================

.. toctree::
   :maxdepth: 4
   :caption: Contents:

   intro
   installation
   examples
   markets
   historical
   time_sequential
   publications

Indices and tables
==================

* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

================
File: docs\source\installation.rst
================
Installation
============
Installing nempy to use in your project is easy.

`pip install nempy`

To install for development purposes, such as adding new features. Download the source code, unzip, cd into the directory, then install.

`pip install e .[dev]`

Then the test suite can be run using.

`python -m pytest`

================
File: docs\source\intro.rst
================
Introduction
============
Nempy is an open-source python package that can be used to model the dispatch procedure of the Australian National
Electricity Market (NEM). The dispatch process is at the core of many market modelling studies. Nempy allows users to
easily configure a dispatch model to fit the relevant research question. Furthermore, if extra functionality is needed,
the python implementation and open-source licencings allow the user to make modifications. Nempy is feature rich,
flexible, can recreate historical dispatch with a high degree of accuracy, runs fast, and has detailed documentation.

The Nempy source code is on GitHub: https://github.com/UNSW-CEEM/nempy.

A brief introduction to the NEM can be found here: https://aemo.com.au/-/media/Files/Electricity/NEM/National-Electricity-Market-Fact-Sheet.pdf

Author
-------
Nempy's development is being led by Nick Gorman as part of his PhD candidature at the Collaboration on Energy and Environmental
Markets at the University of New South Wales' School of Photovoltaics and Renewable Energy Engineering. (https://www.ceem.unsw.edu.au/). 

Support
-------
You can seek support for using Nempy using the discussion tab on GitHub (https://github.com/UNSW-CEEM/nempy/discussions), 
checking the issues register (https://github.com/UNSW-CEEM/nempy/issues), or by contacting Nick directly (n.gorman at unsw.edu.au).

Future support and maintenance
-----------------------------
Planning to continue support and maintenance for Nempy after the PhD project is complete is currently underway. If Nempy is useful 
to your work, research, or business, please reach out and inform us so we can consider your use case and needs.

Example use cases
-----------------
Nempy is intended for analysts and modellers studying the NEM either in industry or academic. It can be
used either as is, or as building block in a large modelling tool. Some potential use case are:

#. As a tool for studying the dispatch process itself. The example shown in the :ref:`section on model accuracy <Accuracy>`
   below demonstrates how model simplifications effects accuracy, this is potentially useful information for other
   NEM modellers either using Nempy or other modelling tools.
#. As a building block in agent based market models, as part of the environment for agents to interact with.
#. To answer counter factual questions about historical dispatch outcomes. For example, how removing a network
   constraint would have effected dispatch and pricing outcomes?
#. As a reference implementation of the NEM's dispatch procedure. Published documentation can lack detail, studying the
   source code of Nempy may be useful for some NEM analysts to gain a better understanding of the dispatch procedure.


Dispatch Procedure Outline
--------------------------
The main task of the dispatch procedure is the construction and solving of a mixed integer linear problem (MIP) to find the
least cost set of dispatch levels for generators and scheduled loads. Note, in this optimisation the dispatch of
scheduled loads is treated as a negative cost, this makes the least cost optimisation equivalent to maximising the value of
market trade. The construction of the MIP as implemented by Nempy proceeds roughly as follows:

#. Bids from generators and loads are preprocessed, some FCAS bids are excluded if they do not meet a set of inclusion
   criteria set out by AEMO (:download:`FCAS Model in NEMDE <../../docs/pdfs/FCAS Model in NEMDE.pdf>`).
#. For each bid a decision variable in the MIP is created, the cost of the variable in the objective function is the bid
   price, and the price is adjusted by a loss factor if one is provided.
#. For each market region a constraint forcing generation to equal demand is created.
#. The rest of the market features are implemented as additional variables and/or constraints in the MIP, for example:

   - unit ramp rates are converted to a set MW ramp that units can achieve over the dispatch interval, and the sum of a
     unit's dispatch is limited by this MW value
   - interconnectors are formulated as additional decision variables that link the supply equals demand constraints
     of the interconnected regions, and are combined with constraints sets that enforce interconnector losses as a
     function of power flow

#. The MIP is solved to determined interconnector flows and dispatch targets, the MIP is then converted to a linear
   problem, and re-solved, such that market prices can be determined from constraint shadow prices.

Differences between Nempy and the dispatch procedure:

#. While updated functionality in Nempy 2.0.0 now provides the capability to calculate RHS values dynamically based on
   SCADA and other data sources, the detailed examples provided for recreating dispatch only calculate RHS values
   relating to the Basslink switch, and other RHS values are taken from the NEMDE solution file.


Features
--------
- **Energy bids**: between one and ten price quantity bid pairs can be provided for each generator or load bidding in the energy market
- **Loss factors**: loss factors can be provided for each generator and load
- **FCAS bids**: between one and ten price quantity bid pairs can be provided for each generator or load bidding in each of the eight FCAS markets
- **Ramp rates**: unit ramp rates can be set
- **FCAS trapezium constraints**: a set of trapezium constraints can be provided for each FCAS bid, these ensure FCAS is co-optimised with energy dispatch and would be technically deliverable
- **Fast start dispatch inflexibility profiles**: dispatch inflexibility profiles can be provided  for unit commitment of fast-start plants
- **Interconnectors and losses**: interconnectors between each market region can be defined, non-linear loss functions and interpolation breakpoints for their linearisation can be provided
- **Generic constraints**: generic constraints that link across unit output, FCAS enablement and interconnector flows can be defined
- **Elastic constraints**: constraints can be made elastic, i.e. a violation cost can be set for constraints
- **Tie-break constraints**: constraints that minimise the difference in dispatch between energy bids for the same price can be enabled
- **Market clearing prices**: market prices are returned for both energy and FCAS markets, based on market constraint shadow prices
- **Historical inputs**: tools for downloading dispatch inputs from AEMO's NEMWeb portal and preprocessing them for compatibility with the nempy SpotMarket class are available
- **Input validation**: optionally check user inputs and raise descriptive errors when they do not meet the expected criteria
- **Adjustable dispatch interval**: a dispatch interval of any length can be used

Flexibility
-----------
Nempy is designed to have a high degree of flexibility, it can be used to implement very simple merit order dispatch models,
highly detailed models that seek to re-create the real world dispatch procedure, or a model at the many levels of intermediate
complexity. A set of :ref:`examples, <examples1>` demonstrating this flexibility are available. Most inputs are passed to nempy as pandas DataFrame
objects, which means Nempy can easily source inputs from other python code, SQL databases, CSVs and other formats supported by
the pandas' interface.

Accuracy
--------
The accuracy with which Nempy represents the NEM's dispatch process can be measured by re-creating historical dispatch results.
This is done for a given dispatch interval by downloading the relevant historical inputs such as unit initial operating levels,
bids and generic constraints, processing these inputs so they are compatible with the Nempy SpotMarket class, and finally
dispatching the spot market. The results can then be compared to historical results to gauge the model's accuracy.
Figure 1 shows the results of this process for 1000 randomly selected dispatch intervals in 2019, comparing the modelled
NSW energy price with historical prices. Here the model is configured to maximally reflect the NEM's dispatch procedure
(not including the Basslink switch run). The code to produce the results shown in this figure is available `here <https://nempy.readthedocs.io/en/latest/publications.html#source-code-for-figure-1>`_.
Figure 2 shows a similar comparison, but without FCAS markets or generic constraints. The code to produce the results
shown in Figure 2 is available `here <https://nempy.readthedocs.io/en/latest/publications.html#source-code-for-figure-2>`_.
The simpler model produces a similar number of medianly priced intervals, however, outcomes for extreme ends of the price
duration curve differ significantly from historical values.

.. image:: nempy_vs_historical.svg
  :width: 600

*Figure 1: A comparison of the historical NSW reference node price, prior to scaling or capping, with the price calculated using nempy.
The nempy model was configured to maximally replicated the NEM dispatch process and 1000 randomly selected intervals were used.*

.. image:: nempy_vs_historical_simple.svg
  :width: 600

*Figure 2: A comparison of the historical NSW reference node price, prior to scaling or capping, with the price calculated
using Nempy. The Nempy model was configured without FCAS markets or generic constraints and 1000 randomly selected intervals were used.*

Run-time
--------
The run-time for Nempy to calculate dispatch depends on several factors, the complexity of the model implemented, time
taken to load inputs, the mixed-integer linear solver used and of course the hardware. Run-times reported here used an
Intel® Xeon(R) W-2145 CPU @ 3.70 GHz. For the model results shown in Figure 1, including time taken to load inputs from
the disk and using the open-source solver CBC, the average run-time per dispatch interval was 2.54 s. When the proprietary
solver Gurobi was used, a run-time of 1.84 s was achieved. For the results shown in Figure 2, the run-times with CBC and
Gurobi were 1.02 s and 0.98 s respectively, indicating that for simpler models the solver used has a smaller impact on
run-time. For the simpler model, the time to load inputs is increased significantly by the loading of historical NEMDE
input/output XML files which takes approximately 0.4 s. Importantly, this means it will be possible to speed up simpler
models by sourcing inputs from different data storage formats.

Notes:

- Information on solvers is provided is provided in the `reference documentation <https://nempy.readthedocs.io/en/latest/markets.html#nempy.markets.SpotMarket.solver_name>`_
  of the SpotMarket class.
- The total runtime was calculated using the python time module and measuring the time taken from the loading of inputs
  to the extraction of results from the model. The runtime of different sub-process, i.e. loading of the XML file, was
  measured by inserting timing code into the Nempy source code where required.

Documentation
-------------
Nempy has a detailed set of documentation, mainly comprising of two types: examples and reference documentation. The
examples aim to show how Nempy can be used and how it works in a practical manner. A number of simple examples focus on
demonstrating the use of subsets of the package's features in isolation in order to make them easier to understand. The
more complex examples show how features can be combined to build models more suitable for analysis. The reference
documentation aims to cover all the package's public APIs (the classes, methods and functions accessible to the user),
describing their use, inputs, outputs and any side effects.

Ongoing work
------------
Enhancements:

* The 1 second raise and lower contingency FCAS markets are in process of being added to Nempy.


Dependencies
------------
* pandas >=1.0.0, <2.0.0
* mip>=1.11.0, <2.0.0: https://github.com/coin-or/python-mip)
* xmltodict==0.12.0:  https://github.com/martinblech/xmltodict)
* requests>=2.0.0, <3.0.0

================
File: docs\source\markets.rst
================
.. _spotmarket:

markets module
===============================
A model of the NEM spot market dispatch process.

Overview
--------
The market, both in real life and in this model, is implemented as a linear program. Linear programs consist of three
elements:

1.  **Decision variables**: the quantities being optimised for. In an electricity market these will be things like the
    outputs of generators, the consumption of dispatchable loads and interconnector flows.
2.  An **objective function**: the linear function being optimised. In this model of the spot market the cost of production
    is being minimised, and is defined as the sum of each bids dispatch level multiplied by the bid price.
3.  A set of **linear constraints**: used to implement market features such as network constraints and interconnectors.

The class :class:`nempy.SpotMarket` is used to construct these elements and then solve the linear program to calculate
dispatch and pricing. The examples below give an overview of how method calls build the linear program.

*   Initialising the market instance, doesn't create any part of the linear program, just saves general information for
    later use.

.. code-block:: python

    market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW'])

*   Providing volume bids creates a set of n decision variables, where n is the number of bids with a volume greater
    than zero.

.. code-block:: python

    market.set_unit_volume_bids(volume_bids)

*   Providing price bids creates the objective function, i.e. units will be dispatch to minimise cost, as determined
    by the bid prices.

.. code-block:: python

    market.set_unit_price_bids(price_bids)

*   Providing unit capacities creates a constraint for each unit that caps its total dispatch at a set capacity

.. code-block:: python

    market.set_unit_bid_capacity_constraints(unit_limits)

*   Providing regional energy demand creates a constraint for each region that forces supply from units and
    interconnectors to equal demand

.. code-block:: python

    market.set_demand_constraints(demand)

Specific examples for using this class are provided on the `examples1`_ page, detailed documentation of the class
:class:`nempy.markets.SpotMarket` is provided in the `Reference`_ material below.

.. _reference:

Reference
---------------------

.. automodule:: nempy.markets
    :autosummary:
    :members:

================
File: docs\source\publications.rst
================
Publications
============
Links to publications and associate source code.

Nempy Technical Brief
---------------------
The nempy technical brief is available :download:`here  <../../docs/pdfs/Nempy Technical Brief v1.0.0.pdf>`.
as pdf, and is also used as the introduction for readthedocs page. The code below uses Nempy v1.1.0 which is
now superseded v2.0.0.

Source code for Figure 1
************************

.. literalinclude:: ../../publications/all_features_example.py
    :linenos:
    :language: python

Source code for Figure 2
************************

.. literalinclude:: ../../publications/energy_only_market.py
    :linenos:
    :language: python

================
File: docs\source\Spot.html
================
<?xml version="1.0" encoding="utf-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="Docutils 0.16: http://docutils.sourceforge.net/" />
<title>The Spot market class</title>
<style type="text/css">

/*
:Author: David Goodger (goodger@python.org)
:Id: $Id: html4css1.css 7952 2016-07-26 18:15:59Z milde $
:Copyright: This stylesheet has been placed in the public domain.

Default cascading style sheet for the HTML output of Docutils.

See http://docutils.sf.net/docs/howto/html-stylesheets.html for how to
customize this style sheet.
*/

/* used to remove borders from tables and images */
.borderless, table.borderless td, table.borderless th {
  border: 0 }

table.borderless td, table.borderless th {
  /* Override padding for "table.docutils td" with "! important".
     The right padding separates the table cells. */
  padding: 0 0.5em 0 0 ! important }

.first {
  /* Override more specific margin styles with "! important". */
  margin-top: 0 ! important }

.last, .with-subtitle {
  margin-bottom: 0 ! important }

.hidden {
  display: none }

.subscript {
  vertical-align: sub;
  font-size: smaller }

.superscript {
  vertical-align: super;
  font-size: smaller }

a.toc-backref {
  text-decoration: none ;
  color: black }

blockquote.epigraph {
  margin: 2em 5em ; }

dl.docutils dd {
  margin-bottom: 0.5em }

object[type="image/svg+xml"], object[type="application/x-shockwave-flash"] {
  overflow: hidden;
}

/* Uncomment (and remove this text!) to get bold-faced definition list terms
dl.docutils dt {
  font-weight: bold }
*/

div.abstract {
  margin: 2em 5em }

div.abstract p.topic-title {
  font-weight: bold ;
  text-align: center }

div.admonition, div.attention, div.caution, div.danger, div.error,
div.hint, div.important, div.note, div.tip, div.warning {
  margin: 2em ;
  border: medium outset ;
  padding: 1em }

div.admonition p.admonition-title, div.hint p.admonition-title,
div.important p.admonition-title, div.note p.admonition-title,
div.tip p.admonition-title {
  font-weight: bold ;
  font-family: sans-serif }

div.attention p.admonition-title, div.caution p.admonition-title,
div.danger p.admonition-title, div.error p.admonition-title,
div.warning p.admonition-title, .code .error {
  color: red ;
  font-weight: bold ;
  font-family: sans-serif }

/* Uncomment (and remove this text!) to get reduced vertical space in
   compound paragraphs.
div.compound .compound-first, div.compound .compound-middle {
  margin-bottom: 0.5em }

div.compound .compound-last, div.compound .compound-middle {
  margin-top: 0.5em }
*/

div.dedication {
  margin: 2em 5em ;
  text-align: center ;
  font-style: italic }

div.dedication p.topic-title {
  font-weight: bold ;
  font-style: normal }

div.figure {
  margin-left: 2em ;
  margin-right: 2em }

div.footer, div.header {
  clear: both;
  font-size: smaller }

div.line-block {
  display: block ;
  margin-top: 1em ;
  margin-bottom: 1em }

div.line-block div.line-block {
  margin-top: 0 ;
  margin-bottom: 0 ;
  margin-left: 1.5em }

div.sidebar {
  margin: 0 0 0.5em 1em ;
  border: medium outset ;
  padding: 1em ;
  background-color: #ffffee ;
  width: 40% ;
  float: right ;
  clear: right }

div.sidebar p.rubric {
  font-family: sans-serif ;
  font-size: medium }

div.system-messages {
  margin: 5em }

div.system-messages h1 {
  color: red }

div.system-message {
  border: medium outset ;
  padding: 1em }

div.system-message p.system-message-title {
  color: red ;
  font-weight: bold }

div.topic {
  margin: 2em }

h1.section-subtitle, h2.section-subtitle, h3.section-subtitle,
h4.section-subtitle, h5.section-subtitle, h6.section-subtitle {
  margin-top: 0.4em }

h1.title {
  text-align: center }

h2.subtitle {
  text-align: center }

hr.docutils {
  width: 75% }

img.align-left, .figure.align-left, object.align-left, table.align-left {
  clear: left ;
  float: left ;
  margin-right: 1em }

img.align-right, .figure.align-right, object.align-right, table.align-right {
  clear: right ;
  float: right ;
  margin-left: 1em }

img.align-center, .figure.align-center, object.align-center {
  display: block;
  margin-left: auto;
  margin-right: auto;
}

table.align-center {
  margin-left: auto;
  margin-right: auto;
}

.align-left {
  text-align: left }

.align-center {
  clear: both ;
  text-align: center }

.align-right {
  text-align: right }

/* reset inner alignment in figures */
div.align-right {
  text-align: inherit }

/* div.align-center * { */
/*   text-align: left } */

.align-top    {
  vertical-align: top }

.align-middle {
  vertical-align: middle }

.align-bottom {
  vertical-align: bottom }

ol.simple, ul.simple {
  margin-bottom: 1em }

ol.arabic {
  list-style: decimal }

ol.loweralpha {
  list-style: lower-alpha }

ol.upperalpha {
  list-style: upper-alpha }

ol.lowerroman {
  list-style: lower-roman }

ol.upperroman {
  list-style: upper-roman }

p.attribution {
  text-align: right ;
  margin-left: 50% }

p.caption {
  font-style: italic }

p.credits {
  font-style: italic ;
  font-size: smaller }

p.label {
  white-space: nowrap }

p.rubric {
  font-weight: bold ;
  font-size: larger ;
  color: maroon ;
  text-align: center }

p.sidebar-title {
  font-family: sans-serif ;
  font-weight: bold ;
  font-size: larger }

p.sidebar-subtitle {
  font-family: sans-serif ;
  font-weight: bold }

p.topic-title {
  font-weight: bold }

pre.address {
  margin-bottom: 0 ;
  margin-top: 0 ;
  font: inherit }

pre.literal-block, pre.doctest-block, pre.math, pre.code {
  margin-left: 2em ;
  margin-right: 2em }

pre.code .ln { color: grey; } /* line numbers */
pre.code, code { background-color: #eeeeee }
pre.code .comment, code .comment { color: #5C6576 }
pre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold }
pre.code .literal.string, code .literal.string { color: #0C5404 }
pre.code .name.builtin, code .name.builtin { color: #352B84 }
pre.code .deleted, code .deleted { background-color: #DEB0A1}
pre.code .inserted, code .inserted { background-color: #A3D289}

span.classifier {
  font-family: sans-serif ;
  font-style: oblique }

span.classifier-delimiter {
  font-family: sans-serif ;
  font-weight: bold }

span.interpreted {
  font-family: sans-serif }

span.option {
  white-space: nowrap }

span.pre {
  white-space: pre }

span.problematic {
  color: red }

span.section-subtitle {
  /* font-size relative to parent (h1..h6 element) */
  font-size: 80% }

table.citation {
  border-left: solid 1px gray;
  margin-left: 1px }

table.docinfo {
  margin: 2em 4em }

table.docutils {
  margin-top: 0.5em ;
  margin-bottom: 0.5em }

table.footnote {
  border-left: solid 1px black;
  margin-left: 1px }

table.docutils td, table.docutils th,
table.docinfo td, table.docinfo th {
  padding-left: 0.5em ;
  padding-right: 0.5em ;
  vertical-align: top }

table.docutils th.field-name, table.docinfo th.docinfo-name {
  font-weight: bold ;
  text-align: left ;
  white-space: nowrap ;
  padding-left: 0 }

/* "booktabs" style (no vertical lines) */
table.docutils.booktabs {
  border: 0px;
  border-top: 2px solid;
  border-bottom: 2px solid;
  border-collapse: collapse;
}
table.docutils.booktabs * {
  border: 0px;
}
table.docutils.booktabs th {
  border-bottom: thin solid;
  text-align: left;
}

h1 tt.docutils, h2 tt.docutils, h3 tt.docutils,
h4 tt.docutils, h5 tt.docutils, h6 tt.docutils {
  font-size: 100% }

ul.auto-toc {
  list-style-type: none }

</style>
</head>
<body>
<div class="document" id="the-spot-market-class">
<h1 class="title">The Spot market class</h1>

<p>A model of the NEM spot market dispatch process.</p>
<div class="section" id="overview">
<h1>Overview</h1>
<p>The market, both in real life and in this model, is implemented as a linear program. Linear programs consist of three
elements:</p>
<ol class="arabic simple">
<li><strong>Decision variables</strong>: the quantities being optimised for. In an electricity market these will be things like the
outputs of generators, the consumption of dispatchable loads and interconnector flows.</li>
<li>An <strong>objective function</strong>: the linear function being optimised. In this model of the spot market the cost of production
is being minimised, and is defined as the sum of each bids dispatch level multiplied by the bid price.</li>
<li>A set of <strong>linear constraints</strong>: used to implement market features such as network constraints and interconnectors.</li>
</ol>
<p>The class <a href="#id1"><span class="problematic" id="id2">:func:`nempy.markets.Spot`</span></a> is used to construct these elements and then solve the linear program to calculate
dispatch and pricing. The examples below give an overview of how method calls build the linear program.</p>
<div class="system-message" id="id1">
<p class="system-message-title">System Message: ERROR/3 (<tt class="docutils">C:/Users/z3417464/Documents/GitHub/nempy/docs/source/Spot.rst</tt>, line 16); <em><a href="#id2">backlink</a></em></p>
Unknown interpreted text role &quot;func&quot;.</div>
<ul class="simple">
<li>Initialising the market instance, doesn't create any part of the linear program, just saves general information for
later use.</li>
</ul>
<pre class="code python literal-block">
<span class="name">market</span> <span class="operator">=</span> <span class="name">markets</span><span class="operator">.</span><span class="name">Spot</span><span class="punctuation">(</span><span class="name">unit_info</span><span class="operator">=</span><span class="name">unit_info</span><span class="punctuation">)</span>
</pre>
<ul class="simple">
<li>Providing volume bids creates a set of n decision variables, where n is the number of bids with a volume greater
than zero.</li>
</ul>
<pre class="code python literal-block">
<span class="name">market</span><span class="operator">.</span><span class="name">set_unit_energy_volume_bids</span><span class="punctuation">(</span><span class="name">volume_bids</span><span class="punctuation">)</span>
</pre>
<ul class="simple">
<li>Providing price bids creates the objective function, i.e. units will be dispatch to minimise cost, as determined
by the bid prices.</li>
</ul>
<pre class="code python literal-block">
<span class="name">market</span><span class="operator">.</span><span class="name">set_unit_energy_price_bids</span><span class="punctuation">(</span><span class="name">price_bids</span><span class="punctuation">)</span>
</pre>
<ul class="simple">
<li>Providing unit capacities creates a constraint for each unit that caps its total dispatch at a set capacity</li>
</ul>
<pre class="code python literal-block">
<span class="name">market</span><span class="operator">.</span><span class="name">set_unit_capacity_constraints</span><span class="punctuation">(</span><span class="name">unit_limits</span><span class="punctuation">)</span>
</pre>
<ul class="simple">
<li>Providing regional energy demand creates a constraint for each region that forces supply from units and
interconnectors to equal demand</li>
</ul>
<pre class="code python literal-block">
<span class="name">market</span><span class="operator">.</span><span class="name">set_demand_constraints</span><span class="punctuation">(</span><span class="name">demand</span><span class="punctuation">)</span>
</pre>
</div>
<div class="section" id="reference">
<h1>Reference</h1>
<div class="system-message">
<p class="system-message-title">System Message: ERROR/3 (<tt class="docutils">C:/Users/z3417464/Documents/GitHub/nempy/docs/source/Spot.rst</tt>, line 56)</p>
<p>Unknown directive type &quot;autoautosummary&quot;.</p>
<pre class="literal-block">
.. autoautosummary::

    .. automethod:: __init__(self, unit_info, dispatch_interval=5)
    .. automethod:: set_unit_energy_volume_bids(self, volume_bids)
    .. automethod:: set_unit_energy_price_bids(self, price_bids)
    .. automethod:: set_unit_capacity_constraints(self, unit_limits)
    .. automethod:: set_unit_ramp_up_constraints(self, unit_limits)
    .. automethod:: set_unit_ramp_down_constraints(self, unit_limits)
    .. automethod:: set_demand_constraints(self, demand)


</pre>
</div>
<div class="system-message">
<p class="system-message-title">System Message: ERROR/3 (<tt class="docutils">C:/Users/z3417464/Documents/GitHub/nempy/docs/source/Spot.rst</tt>, line 67)</p>
<p>Unknown directive type &quot;autoclass&quot;.</p>
<pre class="literal-block">
.. autoclass:: nempy.markets.Spot

    .. automethod:: __init__(self, unit_info, dispatch_interval=5)
    .. automethod:: set_unit_energy_volume_bids(self, volume_bids)
    .. automethod:: set_unit_energy_price_bids(self, price_bids)
    .. automethod:: set_unit_capacity_constraints(self, unit_limits)
    .. automethod:: set_unit_ramp_up_constraints(self, unit_limits)
    .. automethod:: set_unit_ramp_down_constraints(self, unit_limits)
    .. automethod:: set_demand_constraints(self, demand)

</pre>
</div>
</div>
</div>
</body>
</html>

================
File: docs\source\time_sequential.rst
================
.. _time_sequential:

time_sequential modules
=======================
The module provides tools constructing time sequential models using nempy.

.. automodule:: nempy.time_sequential
    :autosummary:
    :members:

================
File: nempy\markets.py
================
import numpy as np
import pandas as pd

from nempy.help_functions import helper_functions as hf
from nempy.spot_markert_backend import elastic_constraints, fcas_constraints, interconnectors as inter, \
    market_constraints, objective_function, solver_interface, unit_constraints, variable_ids, check, \
    dataframe_validator as dv

pd.set_option('display.width', None)


# noinspection PyProtectedMember
class SpotMarket:
    """Class for constructing and dispatching the spot market on an interval basis.

    Examples
    --------
    Define the unit information data needed to initialise the market, in this example all units are in the same
    region.

    >>> unit_info = pd.DataFrame({
    ...     'unit': ['A', 'B'],
    ...     'region': ['NSW', 'NSW']})

    Initialise the market instance.

    >>> market = SpotMarket(market_regions=['NSW'],
    ...                            unit_info=unit_info)

    The units are given a default dispatch_type and loss_factor. Note this data is stored in a private method and
    not intended for public use.

    >>> market._unit_info
      unit region dispatch_type  loss_factor
    0    A    NSW     generator          1.0
    1    B    NSW     generator          1.0

    Parameters
    ----------
    market_regions : list[str]
        The market regions, used to validate inputs.

    unit_info : pd.DataFrame
        Information on a unit basis, not all columns are required.

        =============  ===============================================
        Columns:       Description:
        unit           unique identifier of a dispatch unit,
                       (as `str`)
        region         location of unit, required (as `str`)
        loss_factor    marginal, average or combined loss factors, \n
                       :download:`see AEMO doc <../../docs/pdfs/Treatment_of_Loss_Factors_in_the_NEM.pdf>`, \n
                       optional, (as `np.int64`)
        dispatch_type  "load" or "generator", optional, (as `str`)
        =============  ===============================================

    dispatch_interval : int
        The length of the dispatch interval in minutes, used for interpreting ramp rates.

    Attributes
    ----------
    solver_name : str
        The solver to use must be one of solver options of the mip-python package that is used to interface to solvers.
        Currently the only support solvers are CBC and Gurobi, so allowed solver names are 'CBC' and 'GUROBI'. Default
        value is CBC, CBC works out of the box after installing Nempy, but Gurobi must be installed separately.

    Raises
    ------
        RepeatedRowError
            If there is more than one row for any 'unit'.
        ColumnDataTypeError
            If columns are not of the require type.
        MissingColumnError
            If the column 'units' or 'regions' is missing.
        UnexpectedColumn
            There is a column that is not 'units', 'regions', 'dispatch_type' or 'loss_factor'.
        ColumnValues
            If there are inf, null or negative values in the 'loss_factor' column."""

    def __init__(self, market_regions, unit_info, dispatch_interval=5):
        self.dispatch_interval = dispatch_interval
        self._unit_info = None
        self._decision_variables = {}
        self._variable_to_constraint_map = {'regional': {}, 'unit_level': {}}
        self._constraint_to_variable_map = {'regional': {}, 'unit_level': {}}
        self._lhs_coefficients = {}
        self._generic_constraint_lhs = {}
        self._constraints_rhs_and_type = {}
        self._constraints_dynamic_rhs_and_type = {}
        self._market_constraints_rhs_and_type = {}
        self._objective_function_components = {}
        self._interconnector_directions = None
        self._interconnector_loss_shares = None
        self._next_variable_id = 0
        self._next_constraint_id = 0
        self.validate_inputs = True
        self.check = True
        self._market_regions = market_regions
        self._allowed_dispatch_types = ['generator', 'load']
        self._allowed_services = ['energy', 'raise_reg', 'lower_reg', 'raise_5min', 'lower_5min', 'raise_60s',
                                  'lower_60s', 'raise_6s', 'lower_6s', 'raise_1s', 'lower_1s']
        self._allowed_fcas_services = self._allowed_services[:]
        self._allowed_fcas_services.remove('energy')
        self._allowed_contingency_fcas_services = self._allowed_fcas_services[:]
        self._allowed_contingency_fcas_services.remove('raise_reg')
        self._allowed_contingency_fcas_services.remove('lower_reg')
        self._allowed_regulation_fcas_services = ['raise_reg', 'lower_reg']
        self._allowed_constraint_types = ['<=', '=', '>=']
        self.solver_name = 'CBC'
        self.objective_value = None

        if 'dispatch_type' not in unit_info.columns:
            unit_info['dispatch_type'] = 'generator'

        if 'loss_factor' not in unit_info.columns:
            unit_info['loss_factor'] = 1.0

        if self.validate_inputs:
            self._validate_unit_info(unit_info)

        self._unit_info = unit_info

    def _validate_unit_info(self, unit_info):
        schema = dv.DataFrameSchema(name='unit_info', primary_keys=['unit'])
        schema.add_column(dv.SeriesSchema(name='unit', data_type=str))
        schema.add_column(dv.SeriesSchema(name='region', data_type=str, allowed_values=self._market_regions))
        schema.add_column(dv.SeriesSchema(name='dispatch_type', data_type=str, allowed_values=['generator', 'load']))
        schema.add_column(dv.SeriesSchema(name='loss_factor', data_type=np.float64, must_be_real_number=True,
                                          not_negative=True))
        schema.validate(unit_info)

    def set_unit_volume_bids(self, volume_bids):
        """Creates the decision variables corresponding to unit bids.

        Variables are created by reserving a variable id (as `int`) for each bid. Bids with a volume of 0 MW do not
        have a variable created. The lower bound of the variables are set to zero and the upper bound to the bid
        volume, the variable type is set to continuous. If a no services is specified for the bids they are given the
        default service value of energy is used.

        Examples
        --------

        Define the unit information data set needed to initialise the market, in this example all units are in the same
        region.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                            unit_info=unit_info)

        Define a set of bids, in this example we have two units called A and B, with three bid bands.

        >>> volume_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [20.0, 50.0],
        ...     '2': [20.0, 30.0],
        ...     '3': [5.0, 0.0]})

        Create energy unit bid decision variables.

        >>> market.set_unit_volume_bids(volume_bids)

        The market should now have the variables.

        >>> print(market._decision_variables['bids'])
          unit capacity_band service  variable_id  lower_bound  upper_bound        type
        0    A             1  energy            0          0.0         20.0  continuous
        1    A             2  energy            1          0.0         20.0  continuous
        2    A             3  energy            2          0.0          5.0  continuous
        3    B             1  energy            3          0.0         50.0  continuous
        4    B             2  energy            4          0.0         30.0  continuous

        A mapping of these variables to constraints acting on that unit and service should also exist.

        >>> print(market._variable_to_constraint_map['unit_level']['bids'])
           variable_id unit service  coefficient
        0            0    A  energy          1.0
        1            1    A  energy          1.0
        2            2    A  energy          1.0
        3            3    B  energy          1.0
        4            4    B  energy          1.0

        A mapping of these variables to constraints acting on the units region and service should also exist.

        >>> print(market._variable_to_constraint_map['regional']['bids'])
           variable_id region service  coefficient
        0            0    NSW  energy          1.0
        1            1    NSW  energy          1.0
        2            2    NSW  energy          1.0
        3            3    NSW  energy          1.0
        4            4    NSW  energy          1.0

        Parameters
        ----------
        volume_bids : pd.DataFrame
            Bids by unit, in MW, can contain up to 10 bid bands, these should be labeled '1' to '10'.

            ========  ================================================
            Columns:  Description:
            unit      unique identifier of a dispatch unit (as `str`)
            service   the service being provided, optional, \n
                      default 'energy', (as `str`)
            1         bid volume in the 1st band, in MW, \n
                      (as `np.float64`)
            2         bid volume in the 2nd band, in MW, optional, \n
                      (as `np.float64`)
              :
            10        bid volume in the nth band, in MW, optional, \n
                      (as `np.float64`)
            ========  ================================================

        Returns
        -------
        None

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any unit and service combination.
            ColumnDataTypeError
                If columns are not of the require type.
            MissingColumnError
                If the column 'units' is missing or there are no bid bands.
            UnexpectedColumn
                There is a column that is not 'unit', 'service' or '1' to '10'.
            ColumnValues
                If there are inf, null or negative values in the bid band columns.
        """

        if self.validate_inputs:
            self._validate_volume_bids(volume_bids)

        self._decision_variables['bids'], variable_to_unit_level_constraint_map, variable_to_regional_constraint_map = \
            variable_ids.bids(volume_bids, self._unit_info, self._next_variable_id)

        self._variable_to_constraint_map['regional']['bids'] = variable_to_regional_constraint_map
        self._variable_to_constraint_map['unit_level']['bids'] = variable_to_unit_level_constraint_map
        self._next_variable_id = max(self._decision_variables['bids']['variable_id']) + 1

    def _validate_volume_bids(self, volume_bids):
        schema = dv.DataFrameSchema(name='volume_bids', primary_keys=['unit', 'service'])
        schema.add_column(dv.SeriesSchema(name='unit', data_type=str, allowed_values=self._unit_info['unit']))
        schema.add_column(dv.SeriesSchema(name='service', data_type=str, allowed_values=self._allowed_services),
                          optional=True)
        schema.add_column(dv.SeriesSchema(name=str(1), data_type=np.float64, must_be_real_number=True,
                                          not_negative=True))
        for bid_band in range(2, 11):
            schema.add_column(dv.SeriesSchema(name=str(bid_band), data_type=np.float64, must_be_real_number=True,
                                              not_negative=True), optional=True)
        schema.validate(volume_bids)

    def set_unit_price_bids(self, price_bids):
        """Creates the objective function costs corresponding to energy bids.

        If no loss factors have been provided as part of the unit information when the model was initialised then the
        costs in the objective function are as bid. If loss factors are provided then the bid costs are referred to the
        regional reference node by dividing by the loss factor.

        Examples
        --------

        Define the unit information data set needed to initialise the market, in this example all units are in the same
        region.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info)

        Define a set of bids, in this example we have two units called A and B, with three bid bands.

        >>> volume_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [20.0, 50.0],
        ...     '2': [20.0, 30.0],
        ...     '3': [5.0, 10.0]})

        Create energy unit bid decision variables.

        >>> market.set_unit_volume_bids(volume_bids)

        Define a set of prices for the bids. Bids for each unit need to be monotonically increasing.

        >>> price_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [50.0, 100.0],
        ...     '2': [100.0, 130.0],
        ...     '3': [100.0, 150.0]})

        Create the objective function components corresponding to the energy bids.

        >>> market.set_unit_price_bids(price_bids)

        The variable assocaited with each bid should now have a cost.

        >>> print(market._objective_function_components['bids'])
           variable_id unit service capacity_band   cost
        0            0    A  energy             1   50.0
        1            1    A  energy             2  100.0
        2            2    A  energy             3  100.0
        3            3    B  energy             1  100.0
        4            4    B  energy             2  130.0
        5            5    B  energy             3  150.0

        Parameters
        ----------
        price_bids : pd.DataFrame
            Bids by unit, in $/MW, can contain up to 10 bid bands.

            ========  ================================================
            Columns:  Description:
            unit      unique identifier of a dispatch unit (as `str`)
            service   the service being provided, optional,
                      default 'energy', (as `str`)
            1         bid price in the 1st band, in $/MW, \n
                      (as `np.float64`)
            2         bid price in the 2nd band, in $/MW, optional, \n
                      (as `np.float64`)
                :
            10        bid price in the nth band, in $/MW, optional, \n
                      (as `np.float64`)
            ========  ================================================

        Returns
        -------
        None

        Raises
        ------
            ModelBuildError
                If the volume bids have not been set yet.
            RepeatedRowError
                If there is more than one row for any unit and service combination.
            ColumnDataTypeError
                If columns are not of the required type.
            MissingColumnError
                If the column 'units' is missing or there are no bid bands.
            UnexpectedColumn
                There is a column that is not 'units', 'region' or '1' to '10'.
            ColumnValues
                If there are inf, -inf or null values in the bid band columns.
            BidsNotMonotonicIncreasing
                If the bids band price for all units are not monotonic increasing.
        """
        self._check_unit_volume_bids_set()
        self._validate_price_bids(price_bids)
        energy_objective_function = objective_function.bids(self._decision_variables['bids'], price_bids,
                                                            self._unit_info)
        energy_objective_function = objective_function.scale_by_loss_factors(energy_objective_function, self._unit_info)
        self._objective_function_components['bids'] = \
            energy_objective_function.loc[:, ['variable_id', 'unit', 'service', 'capacity_band', 'cost']]

    def _validate_price_bids(self, price_bids):
        schema = dv.DataFrameSchema(name='price_bids', primary_keys=['unit', 'service'],
                                    row_monatonic_increasing=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'])
        schema.add_column(dv.SeriesSchema(name='unit', data_type=str, allowed_values=self._unit_info['unit']))
        schema.add_column(dv.SeriesSchema(name='service', data_type=str, allowed_values=self._allowed_services),
                          optional=True)
        schema.add_column(dv.SeriesSchema(name=str(1), data_type=np.float64, must_be_real_number=True))
        for bid_band in range(2, 11):
            schema.add_column(dv.SeriesSchema(name=str(bid_band), data_type=np.float64, must_be_real_number=True),
                              optional=True)
        schema.validate(price_bids)

    def _check_unit_volume_bids_set(self):
        if 'bids' not in self._decision_variables:
            raise ModelBuildError('Price bids cannot be set before setting volume bids.')

    def set_unit_bid_capacity_constraints(self, unit_limits):
        """Creates constraints that limit unit output based on their bid in max capacity. If a unit bids in zero
        volume then a constraint is not created.

        Examples
        --------

        Define the unit information data set needed to initialise the market, in this example all units are in the same
        region.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info)

        Define a set of bids, in this example we have two units called A and B, with three bid bands.

        >>> volume_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [20.0, 50.0],
        ...     '2': [20.0, 30.0],
        ...     '3': [5.0, 10.0]})

        Create energy unit bid decision variables.

        >>> market.set_unit_volume_bids(volume_bids)

        Define a set of unit capacities.

        >>> unit_limits = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'capacity': [60.0, 100.0]})

        Create unit capacity based constraints.

        >>> market.set_unit_bid_capacity_constraints(unit_limits)

        The market should now have a set of constraints.

        >>> print(market._constraints_rhs_and_type['unit_bid_capacity'])
          unit service  constraint_id type    rhs
        0    A  energy              0   <=   60.0
        1    B  energy              1   <=  100.0

        ... and a mapping of those constraints to the variable types on the lhs.

        >>> unit_mapping = market._constraint_to_variable_map['unit_level']

        >>> print(unit_mapping['unit_bid_capacity'])
           constraint_id unit service  coefficient
        0              0    A  energy          1.0
        1              1    B  energy          1.0


        Parameters
        ----------
        unit_limits : pd.DataFrame
            Capacity by unit.

            ========  ================================================
            Columns:  Description:
            unit      unique identifier of a dispatch unit (as `str`)
            capacity  The maximum output of the unit if unconstrained \n
                      by ramp rate, in MW (as `np.float64`)
            ========  ================================================

        Returns
        -------
        None

        Raises
        ------
            ModelBuildError
                If the volume bids have not been set yet.
            RepeatedRowError
                If there is more than one row for any unit.
            ColumnDataTypeError
                If columns are not of the required types.
            MissingColumnError
                If the column 'units' or 'capacity' is missing.
            UnexpectedColumn
                There is a column that is not 'units' or 'capacity'.
            ColumnValues
                If there are inf, null or negative values in the bid band columns.
        """
        self._check_unit_volume_bids_set()
        self._validate_unit_limits(unit_limits)
        rhs_and_type, variable_map = unit_constraints.capacity(unit_limits, self._next_constraint_id)
        self._constraints_rhs_and_type['unit_bid_capacity'] = rhs_and_type
        self._constraint_to_variable_map['unit_level']['unit_bid_capacity'] = variable_map
        self._next_constraint_id = max(rhs_and_type['constraint_id']) + 1

    def _validate_unit_limits(self, unit_limits):
        schema = dv.DataFrameSchema(name='unit_limits', primary_keys=['unit'])
        schema.add_column(dv.SeriesSchema(name='unit', data_type=str, allowed_values=self._unit_info['unit']))
        schema.add_column(dv.SeriesSchema(name='capacity', data_type=np.float64))
        schema.validate(unit_limits)

    def set_unconstrained_intermitent_generation_forecast_constraint(self, unit_limits):
        """Creates constraints that limit unit output based on their forecast output.

        Examples
        --------

        Define the unit information data set needed to initialise the market, in this example all units are in the same
        region.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info)

        Define a set of bids, in this example we have two units called A and B, with three bid bands.

        >>> volume_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [20.0, 50.0],
        ...     '2': [20.0, 30.0],
        ...     '3': [5.0, 10.0]})

        Create energy unit bid decision variables.

        >>> market.set_unit_volume_bids(volume_bids)

        Define a set of unit forecast capacities.

        >>> unit_limits = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'capacity': [60.0, 100.0]})

        Create unit capacity based constraints.

        >>> market.set_unconstrained_intermitent_generation_forecast_constraint(unit_limits)

        The market should now have a set of constraints.

        >>> print(market._constraints_rhs_and_type['uigf_capacity'])
          unit service  constraint_id type    rhs
        0    A  energy              0   <=   60.0
        1    B  energy              1   <=  100.0

        ... and a mapping of those constraints to the variable types on the lhs.

        >>> unit_mapping = market._constraint_to_variable_map['unit_level']

        >>> print(unit_mapping['uigf_capacity'])
           constraint_id unit service  coefficient
        0              0    A  energy          1.0
        1              1    B  energy          1.0


        Parameters
        ----------
        unit_limits : pd.DataFrame
            Capacity by unit.

            ========  ================================================
            Columns:  Description:
            unit      unique identifier of a dispatch unit (as `str`)
            capacity  The maximum output of the unit if unconstrained \n
                      by ramp rate, in MW (as `np.float64`)
            ========  ================================================

        Returns
        -------
        None

        Raises
        ------
            ModelBuildError
                If the volume bids have not been set yet.
            RepeatedRowError
                If there is more than one row for any unit.
            ColumnDataTypeError
                If columns are not of the require type.
            MissingColumnError
                If the column 'units' or 'capacity' is missing.
            UnexpectedColumn
                There is a column that is not 'units' or 'capacity'.
            ColumnValues
                If there are inf, null or negative values in the bid band columns.
        """
        self._check_unit_volume_bids_set()
        self._validate_unit_limits(unit_limits)
        rhs_and_type, variable_map = unit_constraints.capacity(unit_limits, self._next_constraint_id)
        self._constraints_rhs_and_type['uigf_capacity'] = rhs_and_type
        self._constraint_to_variable_map['unit_level']['uigf_capacity'] = variable_map
        self._next_constraint_id = max(rhs_and_type['constraint_id']) + 1

    def set_unit_ramp_up_constraints(self, ramp_details):
        """Creates constraints on unit output based on ramp up rate.

        Constrains the unit output to be <= initial_output + ramp_up_rate * (dispatch_interval / 60)

        Examples
        --------
        Define the unit information data set needed to initialise the market, in this example all units are in the same
        region.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info,
        ...                     dispatch_interval=30)

        Define a set of bids, in this example we have two units called A and B, with three bid bands.

        >>> volume_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [20.0, 50.0],
        ...     '2': [20.0, 30.0],
        ...     '3': [5.0, 10.0]})

        Create energy unit bid decision variables.

        >>> market.set_unit_volume_bids(volume_bids)

        Define a set of unit ramp up rates.

        >>> ramp_details = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'initial_output': [20.0, 50.0],
        ...     'ramp_up_rate': [30.0, 100.0]})

        Create unit capacity based constraints.

        >>> market.set_unit_ramp_up_constraints(ramp_details)

        The market should now have a set of constraints.

        >>> print(market._constraints_rhs_and_type['ramp_up'])
          unit service  constraint_id type    rhs
        0    A  energy              0   <=   35.0
        1    B  energy              1   <=  100.0

        ... and a mapping of those constraints to variable type for the lhs.

        >>> unit_mapping = market._constraint_to_variable_map['unit_level']

        >>> print(unit_mapping['ramp_up'])
           constraint_id unit service  coefficient
        0              0    A  energy          1.0
        1              1    B  energy          1.0

        Parameters
        ----------
        ramp_details : pd.DataFrame

            ==============  ==========================================
            Columns:        Description:
            unit            unique identifier of a dispatch unit, \n
                            (as `str`)
            initial_output  the output of the unit at the start of \n
                            the dispatch interval, in MW, \n
                            (as `np.float64`)
            ramp_up_rate    the maximum rate at which the unit can \n
                            increase output, in MW/h, (as `np.float64`)
            ==============  ==========================================

        Returns
        -------
        None

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any unit.
            ColumnDataTypeError
                If columns are not of the require type.
            MissingColumnError
                If the column 'units', 'initial_output' or 'ramp_up_rate' is missing.
            UnexpectedColumn
                There is a column that is not 'units', 'initial_output' or 'ramp_up_rate'.
            ColumnValues
                If there are inf, null or negative values in the bid band columns.
        """
        self._validate_ramp_up_rates(ramp_details)
        rhs_and_type, variable_map = unit_constraints.ramp_up(ramp_details, self._next_constraint_id,
                                                              self.dispatch_interval)
        self._constraints_rhs_and_type['ramp_up'] = rhs_and_type
        self._constraint_to_variable_map['unit_level']['ramp_up'] = variable_map
        self._next_constraint_id = max(rhs_and_type['constraint_id']) + 1

    def _validate_ramp_up_rates(self, ramp_details):
        schema = dv.DataFrameSchema(name='ramp_details', primary_keys=['unit'])
        schema.add_column(dv.SeriesSchema(name='unit', data_type=str, allowed_values=self._unit_info['unit']))
        schema.add_column(dv.SeriesSchema(name='initial_output', data_type=np.float64, must_be_real_number=True))
        schema.add_column(dv.SeriesSchema(name='ramp_up_rate', data_type=np.float64, must_be_real_number=True))
        schema.validate(ramp_details)

    def set_unit_ramp_down_constraints(self, ramp_details):
        """Creates constraints on unit output based on ramp down rate.

        Will constrain the unit output to be >= initial_output - ramp_down_rate * (dispatch_interval / 60).

        Examples
        --------
        Define the unit information data set needed to initialise the market, in this example all units are in the same
        region.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info,
        ...                     dispatch_interval=30)

        Define a set of bids, in this example we have two units called A and B, with three bid bands.

        >>> volume_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [20.0, 50.0],
        ...     '2': [20.0, 30.0],
        ...     '3': [5.0, 10.0]})

        Create energy unit bid decision variables.

        >>> market.set_unit_volume_bids(volume_bids)

        Define a set of unit ramp down rates, also need to provide the initial output of the units at the start of
        dispatch interval.

        >>> ramp_details = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'initial_output': [20.0, 50.0],
        ...     'ramp_down_rate': [20.0, 10.0]})

        Create unit capacity based constraints.

        >>> market.set_unit_ramp_down_constraints(ramp_details)

        The market should now have a set of constraints.

        >>> print(market._constraints_rhs_and_type['ramp_down'])
          unit service  constraint_id type   rhs
        0    A  energy              0   >=  10.0
        1    B  energy              1   >=  45.0

        ... and a mapping of those constraints to variable type for the lhs.

        >>> unit_mapping = market._constraint_to_variable_map['unit_level']

        >>> print(unit_mapping['ramp_down'])
           constraint_id unit service  coefficient
        0              0    A  energy          1.0
        1              1    B  energy          1.0

        Parameters
        ----------
        ramp_details : pd.DataFrame

            ==============  ==========================================
            Columns:        Description:
            unit            unique identifier of a dispatch unit, \n
                            (as `str`)
            initial_output  the output of the unit at the start of \n
                            the dispatch interval, in MW, \n
                            (as `np.float64`)
            ramp_down_rate  the maximum rate at which the unit can, \n
                            decrease output, in MW/h, (as `np.float64`)
            ==============  ==========================================

        Returns
        -------
        None

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any unit.
            ColumnDataTypeError
                If columns are not of the require type.
            MissingColumnError
                If the column 'units', 'initial_output' or 'ramp_down_rate' is missing.
            UnexpectedColumn
                There is a column that is not 'units', 'initial_output' or 'ramp_down_rate'.
            ColumnValues
                If there are inf, null or negative values in the bid band columns.
        """
        self._validate_ramp_down_rates(ramp_details)
        rhs_and_type, variable_map = unit_constraints.ramp_down(ramp_details, self._next_constraint_id,
                                                                self.dispatch_interval)
        self._constraints_rhs_and_type['ramp_down'] = rhs_and_type
        self._constraint_to_variable_map['unit_level']['ramp_down'] = variable_map
        self._next_constraint_id = max(rhs_and_type['constraint_id']) + 1

    def _validate_ramp_down_rates(self, ramp_details):
        schema = dv.DataFrameSchema(name='ramp_details', primary_keys=['unit'])
        schema.add_column(dv.SeriesSchema(name='unit', data_type=str, allowed_values=self._unit_info['unit']))
        schema.add_column(dv.SeriesSchema(name='initial_output', data_type=np.float64, must_be_real_number=True))
        schema.add_column(dv.SeriesSchema(name='ramp_down_rate', data_type=np.float64, must_be_real_number=True,
                                          not_negative=True))
        schema.validate(ramp_details)

    def set_fast_start_constraints(self, fast_start_profiles):
        """Create the constraints on fast start units dispatch, :download:`see AEMO doc <../../docs/pdfs/Fast_Start_Unit_Inflexibility_Profile_Model_October_2014.pdf>`

        Examples
        --------
        Define the unit information data set needed to initialise the market, in this example all units are in the same
        region.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B', 'C', 'D', 'E'],
        ...     'region': ['NSW', 'NSW', 'NSW', 'NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info,
        ...                     dispatch_interval=30)

        Define some example fast start conditions.

        >>> fast_start_conditions = pd.DataFrame({
        ...     'unit': ['A', 'B', 'C', 'D', 'E'],
        ...     'end_mode': [0, 1, 2, 3, 4],
        ...     'time_in_end_mode': [4.0, 5.0, 5.0, 12.0, 10.0],
        ...     'mode_two_length': [7.0, 4.0, 10.0, 8.0, 6.0],
        ...     'mode_four_length': [10.0, 10.0, 20.0, 8.0, 20.0],
        ...     'min_loading': [30.0, 40.0, 35.0, 50.0, 60.0]})

        Add fast start constraints.

        >>> market.set_fast_start_constraints(fast_start_conditions)

        The market should now have a set of constraints.

        >>> print(market._constraints_rhs_and_type['fast_start'])
          unit service  constraint_id type   rhs
        0    A  energy              0   <=   0.0
        1    B  energy              1   <=   0.0
        0    C  energy              2   >=  17.5
        0    C  energy              3   <=  17.5
        0    D  energy              4   >=  50.0
        0    E  energy              5   >=  30.0

        ... and a mapping of those constraints to variable type for the lhs.

        >>> unit_mapping = market._constraint_to_variable_map['unit_level']

        >>> print(unit_mapping['fast_start'])
           constraint_id unit service  coefficient
        0              0    A  energy          1.0
        1              1    B  energy          1.0
        0              3    C  energy          1.0
        0              2    C  energy          1.0
        0              4    D  energy          1.0
        0              5    E  energy          1.0

        Parameters
        ----------
        fast_start_profiles : pd.DataFrame
            ================  ==========================================
            Columns:          Description:
            unit              unique identifier of a dispatch unit, \n
                              (as `str`)
            end_mode          the fast start dispatch mode the unit \n
                              will end the dispatch interval in, \n
                              in minutes, (as `np.int64`),
            time_in_end_mode  the time the unit will have spent in the \n
                              end mode at the end of this dispatch \n
                              interval, in minutes (as `np.int64`)
            mode_two_length   the length of dispatch mode 2 for the \n
                              unit, in minutes, (as `np.int64`)
            mode_four_length  the length of dispatch mode 4 for the \n
                              unit, in minutes, (as `np.int64`)
            min_loading       the minimum stable operating level of \n
                              unit, in MW, (as `np.float64`)
            ================  ==========================================

        Returns
        -------

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any unit.
            ColumnDataTypeError
                If columns are not of the require type.
            MissingColumnError
                If any columns are missing.
            UnexpectedColumn
                If any additional columns are present.
            ColumnValues
                If there are inf, null or negative values in any of the numeric columns.

        """
        if self.validate_inputs:
            self._validate_fast_start_profiles(fast_start_profiles)
        rhs_and_type, variable_map = unit_constraints.create_fast_start_profile_constraints(
            fast_start_profiles, self._next_constraint_id, self.dispatch_interval)
        if not rhs_and_type.empty:
            self._constraints_rhs_and_type['fast_start'] = rhs_and_type
            self._constraint_to_variable_map['unit_level']['fast_start'] = variable_map
            self._next_constraint_id = max(rhs_and_type['constraint_id']) + 1

    def remove_fast_start_constraints(self):
        if 'fast_start' in self._constraints_rhs_and_type:
            del self._constraints_rhs_and_type['fast_start']
            del self._constraint_to_variable_map['unit_level']['fast_start']
        if 'fast_start_deficit' in self._decision_variables:
            del self._decision_variables['fast_start_deficit']
            del self._objective_function_components['fast_start_deficit']
            del self._lhs_coefficients['fast_start_deficit']

    def _validate_fast_start_profiles(self, fast_start_profiles):
        schema = dv.DataFrameSchema(name='fast_start_profiles', primary_keys=['unit'])
        schema.add_column(dv.SeriesSchema(name='unit', data_type=str, allowed_values=self._unit_info['unit']))
        schema.add_column(dv.SeriesSchema(name='end_mode', data_type=np.int64, must_be_real_number=True,
                                          not_negative=True))
        schema.add_column(dv.SeriesSchema(name='time_in_end_mode', data_type=np.float64, must_be_real_number=True,
                                          not_negative=True))
        schema.add_column(dv.SeriesSchema(name='mode_two_length', data_type=np.float64, must_be_real_number=True,
                                          not_negative=True))
        schema.add_column(dv.SeriesSchema(name='mode_four_length', data_type=np.float64, must_be_real_number=True,
                                          not_negative=True))
        schema.add_column(dv.SeriesSchema(name='min_loading', data_type=np.float64, must_be_real_number=True,
                                          not_negative=True))
        schema.validate(fast_start_profiles)

    def set_demand_constraints(self, demand):
        """Creates constraints that force supply to equal to demand.

        Examples
        --------
        Define the unit information data set needed to initialise the market, in this example all units are in the same
        region.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info)

        Define a demand level in each region.

        >>> demand = pd.DataFrame({
        ...     'region': ['NSW'],
        ...     'demand': [100.0]})

        Create constraints.

        >>> market.set_demand_constraints(demand)

        The market should now have a set of constraints.

        >>> print(market._market_constraints_rhs_and_type['demand'])
          region  constraint_id type    rhs
        0    NSW              0    =  100.0

        ... and a mapping of those constraints to variable type for the lhs.

        >>> regional_mapping = market._constraint_to_variable_map['regional']

        >>> print(regional_mapping['demand'])
           constraint_id region service  coefficient
        0              0    NSW  energy          1.0

        Parameters
        ----------
        demand : pd.DataFrame
            Demand by region.

            ========  ================================================
            Columns:  Description:
            region    unique identifier of a region, (as `str`)
            demand    the non dispatchable demand, in MW, \n
                      (as `np.float64`)
            ========  ================================================

        Returns
        -------
        None

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any unit.
            ColumnDataTypeError
                If columns are not of the required type.
            MissingColumnError
                If the column 'region' or 'demand' is missing.
            UnexpectedColumn
                There is a column that is not 'region' or 'demand'.
            ColumnValues
                If there are inf, null or negative values in the volume column.
        """
        if self.validate_inputs:
            self._validate_demand(demand)
        rhs_and_type, variable_map = market_constraints.energy(demand, self._next_constraint_id)
        self._market_constraints_rhs_and_type['demand'] = rhs_and_type
        self._constraint_to_variable_map['regional']['demand'] = variable_map
        self._next_constraint_id = max(rhs_and_type['constraint_id']) + 1

    def _validate_demand(self, demand):
        schema = dv.DataFrameSchema(name='fast_start_profiles', primary_keys=['region'])
        schema.add_column(dv.SeriesSchema(name='region', data_type=str, allowed_values=self._market_regions))
        schema.add_column(dv.SeriesSchema(name='demand', data_type=np.float64, must_be_real_number=True))
        schema.validate(demand)

    def set_fcas_requirements_constraints(self, fcas_requirements):
        """Creates constraints that force FCAS supply to equal requirements.

        Examples
        --------
        Define the unit information data set needed to initialise the market, in this example all units are in the same
        region.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['QLD', 'NSW', 'VIC', 'SA'],
        ...                     unit_info=unit_info)

        Define a regulation raise FCAS requirement that apply to all mainland states.

        >>> fcas_requirements = pd.DataFrame({
        ...     'set': ['raise_reg_main', 'raise_reg_main',
        ...             'raise_reg_main', 'raise_reg_main'],
        ...     'service': ['raise_reg', 'raise_reg',
        ...                 'raise_reg', 'raise_reg'],
        ...     'region': ['QLD', 'NSW', 'VIC', 'SA'],
        ...     'volume': [100.0, 100.0, 100.0, 100.0]})

        Create constraints.

        >>> market.set_fcas_requirements_constraints(fcas_requirements)

        The market should now have a set of constraints.

        >>> print(market._market_constraints_rhs_and_type['fcas'])
                      set  constraint_id type    rhs
        0  raise_reg_main              0    =  100.0

        ... and a mapping of those constraints to variable type for the lhs.

        >>> regional_mapping = \
            market._constraint_to_variable_map['regional']

        >>> print(regional_mapping['fcas'])
           constraint_id    service region  coefficient
        0              0  raise_reg    QLD          1.0
        1              0  raise_reg    NSW          1.0
        2              0  raise_reg    VIC          1.0
        3              0  raise_reg     SA          1.0

        Parameters
        ----------
        fcas_requirements : pd.DataFrame
            requirement by set and the regions and service the requirement applies to.

            ========   ===============================================
            Columns:   Description:
            set        unique identifier of the requirement set, \n
                       (as `str`)
            service    the service or services the requirement set \n
                       applies to (as `str`)
            region     the regions that can contribute to meeting a \n
                       requirement, (as `str`)
            volume     the amount of service required, in MW, \n
                       (as `np.float64`)
            type       the direction of the constrain '=', '>=' or \n
                       '<=', optional, a value of '=' is assumed if \n
                       the column is missing (as `str`)
            ========   ===============================================

        Returns
        -------
        None

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any set, region and service combination.
            ColumnDataTypeError
                If columns are not of the required type.
            MissingColumnError
                If the column 'set', 'service', 'region', or 'volume' is missing.
            UnexpectedColumn
                There is a column that is not 'set', 'service', 'region', 'volume' or 'type'.
            ColumnValues
                If there are inf, null or negative values in the volume column.
        """
        if self.validate_inputs:
            self._validate_fcas_requirements(fcas_requirements)
        rhs_and_type, variable_map = market_constraints.fcas(fcas_requirements, self._next_constraint_id)
        self._market_constraints_rhs_and_type['fcas'] = rhs_and_type
        self._constraint_to_variable_map['regional']['fcas'] = variable_map
        self._next_constraint_id = max(rhs_and_type['constraint_id']) + 1

    def _validate_fcas_requirements(self, fcas_requirements):
        schema = dv.DataFrameSchema(name='fcas_requirements', primary_keys=['set', 'region', 'service'])
        schema.add_column(dv.SeriesSchema(name='region', data_type=str, allowed_values=self._market_regions))
        schema.add_column(dv.SeriesSchema(name='set', data_type=str))
        schema.add_column(dv.SeriesSchema(name='service', data_type=str, allowed_values=self._allowed_services))
        schema.add_column(dv.SeriesSchema(name='volume', data_type=np.float64, must_be_real_number=True))
        schema.add_column(dv.SeriesSchema(name='type', data_type=str, allowed_values=self._allowed_constraint_types),
                          optional=True)
        schema.validate(fcas_requirements)

    def set_fcas_max_availability(self, fcas_max_availability):
        """Creates constraints to ensure fcas dispatch is limited to the availability specified in the FCAS trapezium.

        The constraints are described in the
        :download:`FCAS MODEL IN NEMDE documentation section 2  <../../docs/pdfs/FCAS Model in NEMDE.pdf>`.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info)

        Define the FCAS max_availability.

        >>> fcas_max_availability = pd.DataFrame({
        ... 'unit': ['A'],
        ... 'service': ['raise_6s'],
        ... 'max_availability': [60.0]})

        Set the joint availability constraints.

        >>> market.set_fcas_max_availability(fcas_max_availability)

        TNow the market should have the constraints and their mapping to decision varibales.

        >>> print(market._constraints_rhs_and_type['fcas_max_availability'])
          unit   service  constraint_id type   rhs
        0    A  raise_6s              0   <=  60.0

        >>> unit_mapping = market._constraint_to_variable_map['unit_level']

        >>> print(unit_mapping['fcas_max_availability'])
           constraint_id unit   service  coefficient
        0              0    A  raise_6s          1.0

        Parameters
        ----------
        fcas_max_availability : pd.DataFrame

            ================   =======================================
            Columns:           Description:
            unit               unique identifier of a dispatch unit, \n
                               (as `str`)
            service            the fcas service being offered, \n
                               (as `str`)
            max_availability   the maximum volume of the contingency \n
                               service, in MW, (as `np.float64`)
            ================   =======================================

        Returns
        -------
        None

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any unit and service combination.
            ColumnDataTypeError
                If columns are not of the required type.
            MissingColumnError
                If the columns 'unit', 'service' or 'max_availability' is missing.
            UnexpectedColumn
                If there are columns other than 'unit', 'service' or 'max_availability'.
            ColumnValues
                If there are inf, null or negative values in the columns of type `np.float64`.
        """
        if self.validate_inputs:
            self._validate_fcas_max_availability(fcas_max_availability)
        rhs_and_type, variable_map = unit_constraints.fcas_max_availability(fcas_max_availability,
                                                                            self._next_constraint_id)
        self._constraints_rhs_and_type['fcas_max_availability'] = rhs_and_type
        self._constraint_to_variable_map['unit_level']['fcas_max_availability'] = variable_map
        self._next_constraint_id = max(rhs_and_type['constraint_id']) + 1

    def _validate_fcas_max_availability(self, fcas_max_availability):
        schema = dv.DataFrameSchema(name='fcas_max_availability', primary_keys=['unit', 'service'])
        schema.add_column(dv.SeriesSchema(name='unit', data_type=str, allowed_values=self._unit_info['unit']))
        schema.add_column(dv.SeriesSchema(name='service', data_type=str, allowed_values=self._allowed_fcas_services))
        schema.add_column(dv.SeriesSchema(name='max_availability', data_type=np.float64, must_be_real_number=True,
                                          not_negative=True))
        schema.validate(fcas_max_availability)

    def set_joint_ramping_constraints_raise_reg(self, ramp_details):
        """Create constraints that ensure the provision of energy and fcas raise are within unit ramping capabilities.

        The constraints are described in the
        :download:`FCAS MODEL IN NEMDE documentation section 6.1  <../../docs/pdfs/FCAS Model in NEMDE.pdf>`.

        On a unit basis for generators they take the form of:

            Energy dispatch + Regulation raise target <= initial output + ramp up rate * (dispatch_interval / 60)

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info,
        ...                     dispatch_interval=60)

        Define unit initial outputs and ramping capabilities.

        >>> ramp_details = pd.DataFrame({
        ...   'unit': ['A', 'B'],
        ...   'initial_output': [100.0, 80.0],
        ...   'ramp_up_rate': [20.0, 10.0]})

        Create the joint ramping constraints.

        >>> market.set_joint_ramping_constraints_raise_reg(ramp_details)

        Now the market should have the constraints and their mapping to decision varibales.

        >>> print(market._constraints_rhs_and_type['joint_ramping_raise_reg'])
          unit  constraint_id type    rhs
        0    A              0   <=  120.0
        1    B              1   <=   90.0

        >>> unit_mapping = market._constraint_to_variable_map['unit_level']

        >>> print(unit_mapping['joint_ramping_raise_reg'])
           constraint_id unit    service  coefficient
        0              0    A  raise_reg          1.0
        1              1    B  raise_reg          1.0
        0              0    A     energy          1.0
        1              1    B     energy          1.0

        Parameters
        ----------
        ramp_details : pd.DataFrame

            ==============   =========================================
            Columns:         Description:
            unit             unique identifier of a dispatch unit, \n
                             (as `str`)
            initial_output   the output of the unit at the start of \n
                             the dispatch interval, in MW, \n
                             (as `np.float64`)
            ramp_up_rate     the maximum rate at which the unit can \n
                             increase output, in MW/h, \n
                             (as `np.float64`)
            ==============   =========================================

        Returns
        -------
        None

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any unit in unit_limits.
            ColumnDataTypeError
                If columns are not of the required type.
            MissingColumnError
                If the columns 'unit', 'initial_output' or 'ramp_up_rate' are missing from unit_limits.
            UnexpectedColumn
                If there are columns other than 'unit', 'initial_output' or 'ramp_up_rate' in unit_limits.
            ColumnValues
                If there are inf, null or negative values in the columns of type `np.float64`.
        """
        if self.validate_inputs:
            self._validate_ramp_up_rates(ramp_details)
        ramp_details = ramp_details.rename(columns={'ramp_up_rate': 'ramp_rate'})
        rhs_and_type, variable_map = \
            fcas_constraints.joint_ramping_constraints_raise_reg(ramp_details,
                                                                 self._unit_info.loc[:, ['unit', 'dispatch_type']],
                                                                 self.dispatch_interval, self._next_constraint_id)
        self._constraints_rhs_and_type['joint_ramping_raise_reg'] = rhs_and_type
        self._constraint_to_variable_map['unit_level']['joint_ramping_raise_reg'] = variable_map
        self._next_constraint_id = max(rhs_and_type['constraint_id']) + 1

    def set_joint_ramping_constraints_lower_reg(self, ramp_details):
        """Create constraints that ensure the provision of energy and fcas are within unit ramping capabilities.

        The constraints are described in the
        :download:`FCAS MODEL IN NEMDE documentation section 6.1  <../../docs/pdfs/FCAS Model in NEMDE.pdf>`.

        On a unit basis for generators they take the form of:

            Energy dispatch - Regulation lower target >= initial output - ramp down rate * (dispatch interval / 60)

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info,
        ...                     dispatch_interval=60)

        Define unit initial outputs and ramping capabilities.

        >>> ramp_details = pd.DataFrame({
        ...   'unit': ['A', 'B'],
        ...   'initial_output': [100.0, 80.0],
        ...   'ramp_down_rate': [15.0, 25.0]})

        Create the joint ramping constraints.

        >>> market.set_joint_ramping_constraints_lower_reg(ramp_details)

        Now the market should have the constraints and their mapping to decision varibales.

        >>> print(market._constraints_rhs_and_type['joint_ramping_lower_reg'])
          unit  constraint_id type   rhs
        0    A              0   >=  85.0
        1    B              1   >=  55.0

        >>> print(market._constraint_to_variable_map['unit_level']['joint_ramping_lower_reg'])
           constraint_id unit    service  coefficient
        0              0    A  lower_reg         -1.0
        1              1    B  lower_reg         -1.0
        0              0    A     energy          1.0
        1              1    B     energy          1.0

        Parameters
        ----------
        ramp_details : pd.DataFrame

            ==============  ==========================================
            Columns:        Description:
            unit            unique identifier of a dispatch unit, \n
                            (as `str`)
            initial_output  the output of the unit at the start of, \n
                            the dispatch interval, in MW, \n
                            (as `np.float64`)
            ramp_down_rate  the maximum rate at which the unit can, \n
                            decrease output, in MW/h, \n
                            (as `np.float64`)
            ==============  ==========================================

        Returns
        -------
        None

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any unit in unit_limits.
            ColumnDataTypeError
                If columns are not of the required type.
            MissingColumnError
                If the columns 'unit', 'initial_output' or 'ramp_down_rate' are missing from unit_limits.
            UnexpectedColumn
                If there are columns other than 'unit', 'initial_output' or 'ramp_down_rate' in unit_limits.
            ColumnValues
                If there are inf, null or negative values in the columns of type `np.float64`.
        """

        if self.validate_inputs:
            self._validate_ramp_down_rates(ramp_details)
        ramp_details = ramp_details.rename(columns={'ramp_down_rate': 'ramp_rate'})
        rhs_and_type, variable_map = fcas_constraints.joint_ramping_constraints_lower_reg(
            ramp_details, self._unit_info.loc[:, ['unit', 'dispatch_type']], self.dispatch_interval,
            self._next_constraint_id)

        self._constraints_rhs_and_type['joint_ramping_lower_reg'] = rhs_and_type
        self._constraint_to_variable_map['unit_level']['joint_ramping_lower_reg'] = variable_map
        self._next_constraint_id = max(rhs_and_type['constraint_id']) + 1

    def set_joint_capacity_constraints(self, contingency_trapeziums):
        """Creates constraints to ensure there is adequate capacity for contingency, regulation and energy dispatch.

        Create two constraints for each contingency services, one ensures operation on upper slope of the fcas
        contingency trapezium is consistent with regulation raise and energy dispatch, the second ensures operation on
        upper slope of the fcas contingency trapezium is consistent with regulation lower and energy dispatch.

        The constraints are described in the
        :download:`FCAS MODEL IN NEMDE documentation section 6.2  <../../docs/pdfs/FCAS Model in NEMDE.pdf>`.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A'],
        ...     'region': ['NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info)

        Define the FCAS contingency trapeziums.

        >>> contingency_trapeziums = pd.DataFrame({
        ... 'unit': ['A'],
        ... 'service': ['raise_6s'],
        ... 'max_availability': [60.0],
        ... 'enablement_min': [20.0],
        ... 'low_break_point': [40.0],
        ... 'high_break_point': [60.0],
        ... 'enablement_max': [80.0]})

        Set the joint capacity constraints.

        >>> market.set_joint_capacity_constraints(contingency_trapeziums)

        TNow the market should have the constraints and their mapping to decision varibales.

        >>> print(market._constraints_rhs_and_type['joint_capacity'])
          unit   service  constraint_id type   rhs
        0    A  raise_6s              0   <=  80.0
        0    A  raise_6s              1   >=  20.0

        >>> unit_mapping = market._constraint_to_variable_map['unit_level']

        >>> print(unit_mapping['joint_capacity'])
           constraint_id unit    service  coefficient
        0              0    A     energy     1.000000
        0              0    A   raise_6s     0.333333
        0              0    A  raise_reg     1.000000
        0              1    A     energy     1.000000
        0              1    A   raise_6s    -0.333333
        0              1    A  lower_reg    -1.000000

        Parameters
        ----------
        contingency_trapeziums : pd.DataFrame

            ================   =======================================
            Columns:           Description:
            unit               unique identifier of a dispatch unit, \n
                               (as `str`)
            service            the contingency service being offered, \n
                               (as `str`)
            max_availability   the maximum volume of the contingency \n
                               service, in MW, (as `np.float64`)
            enablement_min     the energy dispatch level at which the \n
                               unit can begin to provide the, \n
                               contingency service, in MW, \n
                               (as `np.float64`)
            low_break_point    the energy dispatch level at which \n
                               the unit can provide the full \n
                               contingency service offered, in MW, \n
                               (as `np.float64`)
            high_break_point   the energy dispatch level at which \n
                               the unit can no longer provide the \n
                               full contingency service offered, \n
                               in MW, (as `np.float64`)
            enablement_max     the energy dispatch level at which \n
                               the unit can no longer provide \n
                               the contingency service, in MW, \n
                               (as `np.float64`)
            ================   =======================================

        Returns
        -------
        None

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any unit and service combination in contingency_trapeziums.
            ColumnDataTypeError
                If columns are not of the required type.
            MissingColumnError
                If the columns 'unit', 'service', 'max_availability', 'enablement_min', 'low_break_point',
                'high_break_point' or 'enablement_max' from contingency_trapeziums.
            UnexpectedColumn
                If there are columns other than 'unit', 'service', 'max_availability', 'enablement_min',
                'low_break_point', 'high_break_point' or 'enablement_max' in contingency_trapeziums.
            ColumnValues
                If there are inf, null or negative values in the columns of type `np.float64`.
        """
        if self.validate_inputs:
            self._validate_contingency_trapeziums(contingency_trapeziums)
        rhs_and_type, variable_map = fcas_constraints.joint_capacity_constraints(
            contingency_trapeziums, self._unit_info.loc[:, ['unit', 'dispatch_type']], self._next_constraint_id)
        self._constraints_rhs_and_type['joint_capacity'] = rhs_and_type
        self._constraint_to_variable_map['unit_level']['joint_capacity'] = variable_map
        self._next_constraint_id = max(rhs_and_type['constraint_id']) + 1

    def _validate_contingency_trapeziums(self, contingency_trapeziums):
        schema = dv.DataFrameSchema(name='contingency_trapeziums', primary_keys=['unit', 'service'])
        schema.add_column(dv.SeriesSchema(name='unit', data_type=str, allowed_values=self._unit_info['unit']))
        schema.add_column(dv.SeriesSchema(name='service', data_type=str,
                                          allowed_values=self._allowed_contingency_fcas_services))
        schema.add_column(dv.SeriesSchema(name='max_availability', data_type=np.float64, must_be_real_number=True,
                                          not_negative=True))
        schema.add_column(dv.SeriesSchema(name='enablement_min', data_type=np.float64, must_be_real_number=True))
        schema.add_column(dv.SeriesSchema(name='low_break_point', data_type=np.float64, must_be_real_number=True))
        schema.add_column(dv.SeriesSchema(name='high_break_point', data_type=np.float64, must_be_real_number=True))
        schema.add_column(dv.SeriesSchema(name='enablement_max', data_type=np.float64, must_be_real_number=True))
        schema.validate(contingency_trapeziums)

    def set_energy_and_regulation_capacity_constraints(self, regulation_trapeziums):
        """Creates constraints to ensure there is adequate capacity for regulation and energy dispatch targets.

        Create two constraints for each regulation services, one ensures operation on upper slope of the fcas
        regulation trapezium is consistent with energy dispatch, the second ensures operation on lower slope of the
        fcas regulation trapezium is consistent with energy dispatch.

        The constraints are described in the
        :download:`FCAS MODEL IN NEMDE documentation section 6.3  <../../docs/pdfs/FCAS Model in NEMDE.pdf>`.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A'],
        ...     'region': ['NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info)

        Define the FCAS regulation trapeziums.

        >>> regulation_trapeziums = pd.DataFrame({
        ... 'unit': ['A'],
        ... 'service': ['raise_reg'],
        ... 'max_availability': [60.0],
        ... 'enablement_min': [20.0],
        ... 'low_break_point': [40.0],
        ... 'high_break_point': [60.0],
        ... 'enablement_max': [80.0]})

        Set the joint capacity constraints.

        >>> market.set_energy_and_regulation_capacity_constraints(regulation_trapeziums)

        TNow the market should have the constraints and their mapping to decision varibales.

        >>> print(market._constraints_rhs_and_type['energy_and_regulation_capacity'])
          unit    service  constraint_id type   rhs
        0    A  raise_reg              0   <=  80.0
        0    A  raise_reg              1   >=  20.0

        >>> unit_mapping = market._constraint_to_variable_map['unit_level']

        >>> print(unit_mapping['energy_and_regulation_capacity'])
           constraint_id unit    service  coefficient
        0              0    A     energy     1.000000
        0              0    A  raise_reg     0.333333
        0              1    A     energy     1.000000
        0              1    A  raise_reg    -0.333333

        Parameters
        ----------
        regulation_trapeziums : pd.DataFrame
            The FCAS trapeziums for the regulation services being offered.

            ================   =======================================
            Columns:           Description:
            unit               unique identifier of a dispatch unit, \n
                               (as `str`)
            service            the regulation service being offered, \n
                               (as `str`)
            max_availability   the maximum volume of the contingency \n
                               service, in MW, (as `np.float64`)
            enablement_min     the energy dispatch level at which \n
                               the unit can begin to provide \n
                               the regulation service, in MW, \n
                               (as `np.float64`)
            low_break_point    the energy dispatch level at which \n
                               the unit can provide the full \n
                               regulation service offered, in MW, \n
                               (as `np.float64`)
            high_break_point   the energy dispatch level at which the \n
                               unit can no longer provide the \n
                               full regulation service offered, in MW, \n
                               (as `np.float64`)
            enablement_max     the energy dispatch level at which the \n
                               unit can no longer provide any \n
                               regulation service, in MW, \n
                               (as `np.float64`)
            ================   =======================================

        Returns
        -------
        None

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any unit and service combination in regulation_trapeziums.
            ColumnDataTypeError
                If columns are not of the required type.
            MissingColumnError
                If the columns 'unit', 'service', 'max_availability', 'enablement_min', 'low_break_point',
                'high_break_point' or 'enablement_max' from regulation_trapeziums.
            UnexpectedColumn
                If there are columns other than 'unit', 'service', 'max_availability', 'enablement_min',
                'low_break_point', 'high_break_point' or 'enablement_max' in regulation_trapeziums.
            ColumnValues
                If there are inf, null or negative values in the columns of type `np.float64`.
        """
        if variable_ids:
            self._validate_regulation_trapeziums(regulation_trapeziums)
        rhs_and_type, variable_map = \
            fcas_constraints.energy_and_regulation_capacity_constraints(regulation_trapeziums, self._next_constraint_id)
        self._constraints_rhs_and_type['energy_and_regulation_capacity'] = rhs_and_type
        self._constraint_to_variable_map['unit_level']['energy_and_regulation_capacity'] = variable_map
        self._next_constraint_id = max(rhs_and_type['constraint_id']) + 1

    def _validate_regulation_trapeziums(self, contingency_trapeziums):
        schema = dv.DataFrameSchema(name='contingency_trapeziums', primary_keys=['unit', 'service'])
        schema.add_column(dv.SeriesSchema(name='unit', data_type=str, allowed_values=self._unit_info['unit']))
        schema.add_column(dv.SeriesSchema(name='service', data_type=str,
                                          allowed_values=self._allowed_regulation_fcas_services))
        schema.add_column(dv.SeriesSchema(name='max_availability', data_type=np.float64, must_be_real_number=True,
                                          not_negative=True))
        schema.add_column(dv.SeriesSchema(name='enablement_min', data_type=np.float64, must_be_real_number=True))
        schema.add_column(dv.SeriesSchema(name='low_break_point', data_type=np.float64, must_be_real_number=True))
        schema.add_column(dv.SeriesSchema(name='high_break_point', data_type=np.float64, must_be_real_number=True))
        schema.add_column(dv.SeriesSchema(name='enablement_max', data_type=np.float64, must_be_real_number=True))
        schema.validate(contingency_trapeziums)

    def set_interconnectors(self, interconnector_directions_and_limits):
        """Create lossless links between specified regions.

        Examples
        --------

        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A'],
        ...     'region': ['NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW', 'VIC'],
        ...                     unit_info=unit_info)

        Define a an interconnector between NSW and VIC so generator can A can be used to meet demand in VIC.

        >>> interconnector = pd.DataFrame({
        ...     'interconnector': ['inter_one'],
        ...     'to_region': ['VIC'],
        ...     'from_region': ['NSW'],
        ...     'max': [100.0],
        ...     'min': [-100.0]})

        Create the interconnector.

        >>> market.set_interconnectors(interconnector)

        The market should now have a decision variable defined for each interconnector.

        >>> print(market._decision_variables['interconnectors'])
          interconnector       link  variable_id  lower_bound  upper_bound        type  generic_constraint_factor
        0      inter_one  inter_one            0       -100.0        100.0  continuous                          1

        ... and a mapping of those variables to to regional energy constraints.

        >>> regional = market._variable_to_constraint_map['regional']

        >>> print(regional['interconnectors'])
           variable_id interconnector       link region service  coefficient
        0            0      inter_one  inter_one    VIC  energy          1.0
        1            0      inter_one  inter_one    NSW  energy         -1.0

        Parameters
        ----------
        interconnector_directions_and_limits : pd.DataFrame

            ========================  ================================
            Columns:                  Description:
            interconnector            unique identifier of a  \n
                                      interconnector, (as `str`)
            to_region                 the region that receives power \n
                                      when flow is in the positive \n
                                      direction, (as `str`)
            from_region               the region that power is drawn \n
                                      from when flow is in the \n
                                      positive direction, (as `str`)
            max                       the maximum power flow in the \n
                                      positive direction, in MW, \n
                                      (as `np.float64`)
            min                       the maximum power flow in the \n
                                      negative direction, in MW, \n
                                      (as `np.float64`)
            from_region_loss_factor   the loss factor at the from \n
                                      region end of the interconnector, \n
                                      refers the the from region end \n
                                      to the regional reference node, \n
                                      optional, assumed to equal 1.0, \n
                                      if the column is not provided, \n
                                      (as `np.float`)
            to_region_loss_factor     the loss factor at the to region \n
                                      end of the interconnector, \n
                                      refers the to region end to the \n
                                      regional reference node, \n
                                      optional, assumed equal to 1.0 \n
                                      if the column is not provided, \n
                                      (as `np.float`)
            ========================  ================================

        Returns
        -------
        None

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any interconnector.
            ColumnDataTypeError
                If columns are not of the require type.
            MissingColumnError
                If any columns are missing.
            UnexpectedColumn
                If there are any additional columns in the input DataFrame.
            ColumnValues
                If there are inf, null values in the max and min columns.
        """
        if 'link' not in interconnector_directions_and_limits.columns:
            interconnector_directions_and_limits['link'] = interconnector_directions_and_limits['interconnector']
        if 'from_region_loss_factor' not in interconnector_directions_and_limits.columns:
            interconnector_directions_and_limits['from_region_loss_factor'] = 1.0
        if 'to_region_loss_factor' not in interconnector_directions_and_limits.columns:
            interconnector_directions_and_limits['to_region_loss_factor'] = 1.0
        if 'generic_constraint_factor' not in interconnector_directions_and_limits.columns:
            interconnector_directions_and_limits['generic_constraint_factor'] = 1

        if self.validate_inputs:
            self._validate_interconnector_definitions(interconnector_directions_and_limits)

        self._interconnector_directions = interconnector_directions_and_limits

        self._decision_variables['interconnectors'], self._variable_to_constraint_map['regional']['interconnectors'] \
            = inter.create(interconnector_directions_and_limits, self._next_variable_id)

        self._next_variable_id = max(self._decision_variables['interconnectors']['variable_id']) + 1

    def _validate_interconnector_definitions(self, interconnector_directions_and_limits):
        schema = dv.DataFrameSchema(name='interconnector_directions_and_limits',
                                    primary_keys=['interconnector', 'link'])
        schema.add_column(dv.SeriesSchema(name='interconnector', data_type=str))
        schema.add_column(dv.SeriesSchema(name='link', data_type=str))
        schema.add_column(dv.SeriesSchema(name='to_region', data_type=str, allowed_values=self._market_regions))
        schema.add_column(dv.SeriesSchema(name='from_region', data_type=str, allowed_values=self._market_regions))
        schema.add_column(dv.SeriesSchema(name='max', data_type=np.float64, must_be_real_number=True))
        schema.add_column(dv.SeriesSchema(name='min', data_type=np.float64, must_be_real_number=True))
        schema.add_column(dv.SeriesSchema(name='generic_constraint_factor', data_type=np.int64, allowed_values=[1, -1]))
        schema.add_column(dv.SeriesSchema(name='from_region_loss_factor', data_type=np.float64,
                                          must_be_real_number=True, not_negative=True))
        schema.add_column(dv.SeriesSchema(name='to_region_loss_factor', data_type=np.float64, must_be_real_number=True,
                                          not_negative=True))
        schema.validate(interconnector_directions_and_limits)

    def set_interconnector_losses(self, loss_functions, interpolation_break_points):
        """Creates linearised loss functions for interconnectors.

        Creates a loss variable for each interconnector, this variable models losses by adding demand to each region.
        The losses are proportioned to each region according to the from_region_loss_share. In a region with one
        interconnector, where the region is the nominal from region, the impact on the demand constraint would be:

            generation - interconnector flow - interconnector losses * from_region_loss_share = demand

        If the region was the nominal to region, then:

            generation + interconnector flow - interconnector losses *  (1 - from_region_loss_share) = demand

        The loss variable is constrained to be a linear interpolation of the loss function between the two break points
        either side of to the actual line flow. This is achieved using a type 2 Special ordered set, where each
        variable is bound between 0 and 1, only 2 variables can be greater than 0 and all variables must sum to 1.
        The actual loss function is evaluated at each break point, the variables of the special order set are
        constrained such that their values weight the distance of the actual flow from the break points on either side
        e.g. If we had 3 break points at -100 MW, 0 MW and 100 MW, three weight variables w1, w2, and w3,
        and a loss function f, then the constraints would be of the form.

        Constrain the weight variables to sum to one:

            w1 + w2 + w3 = 1

        Constrain the weight variables to give the relative weighting of adjacent breakpoint:

            w1 * -100.0 + w2 * 0.0 + w3 * 100.0 = interconnector flow

        Constrain the interconnector losses to be the weighted sum of the losses at the adjacent break point:

            w1 * f(-100.0) + w2 * f(0.0) + w3 * f(100.0) = interconnector losses

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A'],
        ...     'region': ['NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW', 'VIC'],
        ...                     unit_info=unit_info)

        Create the interconnector, this need to be done before a interconnector losses can be set.

        >>> interconnectors = pd.DataFrame({
        ...    'interconnector': ['little_link'],
        ...    'to_region': ['VIC'],
        ...    'from_region': ['NSW'],
        ...    'max': [100.0],
        ...    'min': [-120.0]})

        >>> market.set_interconnectors(interconnectors)

        Define the interconnector loss function. In this case losses are always 5 % of line flow.

        >>> def constant_losses(flow):
        ...     return abs(flow) * 0.05

        Define the function on a per interconnector basis. Also details how the losses should be proportioned to the
        connected regions.

        >>> loss_functions = pd.DataFrame({
        ...    'interconnector': ['little_link'],
        ...    'from_region_loss_share': [0.5],  # losses are shared equally.
        ...    'loss_function': [constant_losses]})

        Define The points to linearly interpolate the loss function between. In this example the loss function is
        linear so only three points are needed, but if a non linear loss function was used then more points would
        result in a better approximation.

        >>> interpolation_break_points = pd.DataFrame({
        ...    'interconnector': ['little_link', 'little_link', 'little_link'],
        ...    'loss_segment': [1, 2, 3],
        ...    'break_point': [-120.0, 0.0, 100]})

        >>> market.set_interconnector_losses(loss_functions, interpolation_break_points)

        The market should now have a decision variable defined for each interconnector's losses.

        >>> print(market._decision_variables['interconnector_losses'])
          interconnector         link  variable_id  lower_bound  upper_bound        type
        0    little_link  little_link            1       -120.0        120.0  continuous

        ... and a mapping of those variables to regional energy constraints.

        >>> print(market._variable_to_constraint_map['regional']['interconnector_losses'])
           variable_id region service  coefficient
        0            1    VIC  energy         -0.5
        1            1    NSW  energy         -0.5

        The market will also have a special ordered set of weight variables for interpolating the loss function
        between the break points.

        >>> print(market._decision_variables['interpolation_weights'].loc[:,
        ...       ['interconnector', 'loss_segment', 'break_point', 'variable_id']])
          interconnector  loss_segment  break_point  variable_id
        0    little_link             1       -120.0            2
        1    little_link             2          0.0            3
        2    little_link             3        100.0            4

        >>> print(market._decision_variables['interpolation_weights'].loc[:,
        ...       ['variable_id', 'lower_bound', 'upper_bound', 'type']])
           variable_id  lower_bound  upper_bound        type
        0            2          0.0          1.0  continuous
        1            3          0.0          1.0  continuous
        2            4          0.0          1.0  continuous

        and a set of constraints that implement the interpolation, see above explanation.

        >>> print(market._constraints_rhs_and_type['interpolation_weights'])
          interconnector         link  constraint_id type  rhs
        0    little_link  little_link              0    =  1.0

        >>> print(market._constraints_dynamic_rhs_and_type['link_loss_to_flow'])
          interconnector         link  constraint_id type  rhs_variable_id
        0    little_link  little_link              2    =                0
        0    little_link  little_link              1    =                1

        >>> print(market._lhs_coefficients['interconnector_losses'])
           variable_id  constraint_id  coefficient
        0            2              0          1.0
        1            3              0          1.0
        2            4              0          1.0
        0            2              2       -120.0
        1            3              2          0.0
        2            4              2        100.0
        0            2              1          6.0
        1            3              1          0.0
        2            4              1          5.0


        Parameters
        ----------
        loss_functions : pd.DataFrame

            ======================  ==================================
            Columns:                Description:
            interconnector          unique identifier of a \n
                                    interconnector, (as `str`)
            from_region_loss_share  The fraction of loss occuring in \n
                                    the from region, 0.0 to 1.0, \n
                                    (as `np.float64`)
            loss_function           A function that takes a flow, \n
                                    in MW as a float and returns the \n
                                    losses in MW, (as `callable`)
            ======================  ==================================

        interpolation_break_points : pd.DataFrame

            ==============  ==========================================
            Columns:        Description:
            interconnector  unique identifier of a interconnector, \n
                            (as `str`)
            loss_segment    unique identifier of a loss segment on \n
                            an interconnector basis, (as `np.float64`)
            break_point     points between which the loss function \n
                            will be linearly interpolated, in MW, \n
                            (as `np.float64`)
            ==============  ==========================================

        Returns
        -------
        None

        Raises
        ------
            ModelBuildError
                If all the interconnectors in the input data have not already been added to the model.
            RepeatedRowError
                If there is more than one row for any interconnector in loss_functions. Or if there is a repeated break
                point for an interconnector in interpolation_break_points.
            ColumnDataTypeError
                If columns are not of the required type.
            MissingColumnError
                If any columns are missing.
            UnexpectedColumn
                If there are any additional columns in the input DataFrames.
            ColumnValues
                If there are inf or null values in the numeric columns of either input DataFrames. Or if
                from_region_loss_share are outside the range of 0.0 to 1.0
        """
        if 'interconnectors' not in self._decision_variables:
            ModelBuildError('Interconnector losses cannot be set before interconnectors have been added to the model.')

        if 'link' not in loss_functions.columns:
            loss_functions['link'] = loss_functions['interconnector']

        if 'link' not in interpolation_break_points.columns:
            interpolation_break_points['link'] = interpolation_break_points['interconnector']

        if self.validate_inputs:
            self._validate_loss_functions(loss_functions)
            self._validate_interpolation_break_points(interpolation_break_points)

        self._interconnector_loss_shares = loss_functions.loc[:, ['interconnector', 'link', 'from_region_loss_share']]

        loss_functions = pd.merge(loss_functions,
                                  self._interconnector_directions.loc[:, ['interconnector', 'link', 'from_region']],
                                  on=['interconnector', 'link'])

        loss_variables, loss_variables_constraint_map = \
            inter.create_loss_variables(self._decision_variables['interconnectors'],
                                        self._variable_to_constraint_map['regional']['interconnectors'],
                                        loss_functions,
                                        self._next_variable_id)

        next_variable_id = loss_variables['variable_id'].max() + 1

        weight_variables = inter.create_weights(interpolation_break_points, next_variable_id)

        # Creates weights sum constraint.
        weights_sum_lhs, weights_sum_rhs = inter.create_weights_must_sum_to_one(weight_variables,
                                                                                self._next_constraint_id)
        next_constraint_id = weights_sum_rhs['constraint_id'].max() + 1

        # Link the losses to the interpolation weights.
        link_to_loss_lhs, link_to_loss_rhs = \
            inter.link_inter_loss_to_interpolation_weights(weight_variables, loss_variables, loss_functions,
                                                           next_constraint_id)
        next_constraint_id = link_to_loss_rhs['constraint_id'].max() + 1

        # Link weights to interconnector flow.
        link_to_flow_lhs, link_to_flow_rhs = inter.link_weights_to_inter_flow(weight_variables,
                                                                              self._decision_variables[
                                                                                  'interconnectors'],
                                                                              next_constraint_id)

        # Combine lhs sides, note these are complete lhs and don't need to be mapped to constraints.
        lhs = pd.concat([weights_sum_lhs, link_to_flow_lhs, link_to_loss_lhs])

        # Combine constraints with a dynamic rhs i.e. a variable on the rhs.
        dynamic_rhs = pd.concat([link_to_flow_rhs, link_to_loss_rhs])

        # Save results.
        self._decision_variables['interconnector_losses'] = loss_variables
        self._variable_to_constraint_map['regional']['interconnector_losses'] = loss_variables_constraint_map
        self._decision_variables['interpolation_weights'] = weight_variables
        self._lhs_coefficients['interconnector_losses'] = lhs
        self._constraints_rhs_and_type['interpolation_weights'] = weights_sum_rhs
        self._constraints_dynamic_rhs_and_type['link_loss_to_flow'] = dynamic_rhs
        self._next_variable_id = pd.concat([loss_variables, weight_variables])['variable_id'].max() + 1
        self._next_constraint_id = pd.concat([weights_sum_rhs, dynamic_rhs])['constraint_id'].max() + 1

    @staticmethod
    def _validate_loss_functions(loss_functions):
        schema = dv.DataFrameSchema(name='loss_functions', primary_keys=['interconnector', 'link'])
        schema.add_column(dv.SeriesSchema(name='interconnector', data_type=str))
        schema.add_column(dv.SeriesSchema(name='link', data_type=str))
        schema.add_column(dv.SeriesSchema(name='loss_function', data_type=callable))
        schema.add_column(dv.SeriesSchema(name='from_region_loss_share', data_type=np.float64, must_be_real_number=True,
                                          not_negative=True))
        schema.validate(loss_functions)

    @staticmethod
    def _validate_interpolation_break_points(interpolation_break_points):
        schema = dv.DataFrameSchema(name='interpolation_break_points', primary_keys=['interconnector', 'link',
                                                                                     'loss_segment'])
        schema.add_column(dv.SeriesSchema(name='interconnector', data_type=str))
        schema.add_column(dv.SeriesSchema(name='link', data_type=str))
        schema.add_column(dv.SeriesSchema(name='loss_segment', data_type=np.int64))
        schema.add_column(dv.SeriesSchema(name='break_point', data_type=np.float64, must_be_real_number=True))
        schema.validate(interpolation_break_points)

    def set_generic_constraints(self, generic_constraint_parameters):
        """Creates a set of generic constraints, adding the constraint type, rhs.

        This sets a set of arbitrary constraints, but only the type and rhs values. The lhs terms can be added to these
        constraints using the methods link_units_to_generic_constraints, link_interconnectors_to_generic_constraints
        and link_regions_to_generic_constraints.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A'],
        ...     'region': ['NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info)

        Define a set of generic constraints and add them to the market.

        >>> generic_constraint_parameters = pd.DataFrame({
        ...   'set': ['A', 'B'],
        ...   'type': ['>=', '<='],
        ...   'rhs': [10.0, -100.0]})

        >>> market.set_generic_constraints(generic_constraint_parameters)

        Now the market should have a set of generic constraints.

        >>> print(market._constraints_rhs_and_type['generic'])
          set  constraint_id type    rhs
        0   A              0   >=   10.0
        1   B              1   <= -100.0

        Parameters
        ----------
        generic_constraint_parameters : pd.DataFrame

            =============  ===========================================
            Columns:       Description:
            set            the unique identifier of the constraint set, \n
                           (as `str`)
            type           the direction of the constraint >=, <= or \n
                           =, (as `str`)
            rhs            the right hand side value of the constraint, \n
                           (as `np.float64`)
            =============  ===========================================

        Returns
        -------
        None

        Raises
        ------
            RepeatedRowError
                If there is more than one row for any unit.
            ColumnDataTypeError
                If columns are not of the required type.
            MissingColumnError
                If the column 'set', 'type' or 'rhs' is missing.
            UnexpectedColumn
                There is a column that is not 'set', 'type' or 'rhs' .
            ColumnValues
                If there are inf or null values in the rhs column.
        """
        if self.validate_inputs:
            self._validate_generic_constraint_parameters(generic_constraint_parameters)
        type_and_rhs = hf.save_index(generic_constraint_parameters, 'constraint_id', self._next_constraint_id)
        self._constraints_rhs_and_type['generic'] = type_and_rhs.loc[:, ['set', 'constraint_id', 'type', 'rhs']]
        self._next_constraint_id = type_and_rhs['constraint_id'].max() + 1

    @staticmethod
    def _validate_generic_constraint_parameters(generic_constraint_parameters):
        schema = dv.DataFrameSchema(name='generic_constraint_parameters', primary_keys=['set'])
        schema.add_column(dv.SeriesSchema(name='set', data_type=str))
        schema.add_column(dv.SeriesSchema(name='type', data_type=str))
        schema.add_column(dv.SeriesSchema(name='rhs', data_type=np.float64, must_be_real_number=True))
        schema.validate(generic_constraint_parameters)

    def link_units_to_generic_constraints(self, unit_coefficients):
        """Set the lhs coefficients of generic constraints on unit basis.

        Notes
        -----
        These sets also maps to the sets in the fcas market constraints. One potential use of this is prevent specific
        units from helping to meet fcas constraints by giving them a negative one (-1.0) coefficient using this method
        for particular fcas markey constraints.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'X', 'Y'],
        ...     'region': ['NSW', 'NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW', 'VIC'],
        ...                     unit_info=unit_info)

        Define unit lhs coefficients for generic constraints.

        >>> unit_coefficients = pd.DataFrame({
        ...   'set': ['A', 'A', 'B'],
        ...   'unit': ['X', 'Y', 'X'],
        ...   'service': ['energy', 'energy', 'raise_reg'],
        ...   'coefficient': [1.0, 1.0, -1.0]})

        >>> market.link_units_to_generic_constraints(unit_coefficients)

        Note all this does is save this information to the market object, linking to specific variable ids and
        constraint id occurs when the dispatch method is called.

        >>> print(market._generic_constraint_lhs['unit'])
          set unit    service  coefficient
        0   A    X     energy          1.0
        1   A    Y     energy          1.0
        2   B    X  raise_reg         -1.0

        Parameters
        ----------
        unit_coefficients : pd.DataFrame

            =============  ===========================================
            Columns:       Description:
            set            the unique identifier of the constraint set \n
                           to map the lhs coefficients to, (as `str`)
            unit           the unit whose variables will be mapped to \n
                           the lhs, (as `str`)
            service        the service whose variables will be mapped
                           to the lhs, (as `str`)
            coefficient    the lhs coefficient (as `np.float64`)
            =============  ===========================================

        Raises
        ------
        RepeatedRowError
            If there is more than one row for any set, unit and service combination.
        ColumnDataTypeError
            If columns are not of the required type.
        MissingColumnError
            If the column 'set', 'unit', 'serice' or 'coefficient' is missing.
        UnexpectedColumn
            There is a column that is not 'set', 'unit', 'serice' or 'coefficient'.
        ColumnValues
            If there are inf or null values in the rhs coefficient.
        """
        if self.validate_inputs:
            self._validate_generic_unit_coefficients(unit_coefficients)
        self._generic_constraint_lhs['unit'] = unit_coefficients

    def _validate_generic_unit_coefficients(self, unit_coefficients):
        schema = dv.DataFrameSchema(name='unit_coefficients', primary_keys=['set', 'unit', 'service'])
        schema.add_column(dv.SeriesSchema(name='set', data_type=str))
        schema.add_column(dv.SeriesSchema(name='unit', data_type=str, allowed_values=list(self._unit_info['unit'])))
        schema.add_column(dv.SeriesSchema(name='service', data_type=str, allowed_values=self._allowed_services))
        schema.add_column(dv.SeriesSchema(name='coefficient', data_type=np.float64, must_be_real_number=True))
        schema.validate(unit_coefficients)

    def link_regions_to_generic_constraints(self, region_coefficients):
        """Set the lhs coefficients of generic constraints on region basis.

        This effectively acts as short cut for mapping unit variables to a generic constraint. If a particular
        service in a particular region is included here then all units in this region will have their variables
        of this service included on the lhs of this constraint set. If a particular unit needs to be excluded
        from an otherwise region wide constraint it can be given a coefficient with opposite sign to the region
        wide sign in the generic unit constraints, the coefficients from the two lhs set will be summed and cancel
        each other out.

        Notes
        -----
        These sets also map to the sets in the fcas market constraints.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['X', 'X']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['X', 'Y'],
        ...                     unit_info=unit_info)

        Define region lhs coefficients for generic constraints.

        >>> region_coefficients = pd.DataFrame({
        ...   'set': ['A', 'A', 'B'],
        ...   'region': ['X', 'Y', 'X'],
        ...   'service': ['energy', 'energy', 'raise_reg'],
        ...   'coefficient': [1.0, 1.0, -1.0]})

        >>> market.link_regions_to_generic_constraints(region_coefficients)

        Note all this does is save this information to the market object, linking to specific variable ids and
        constraint id occurs when the dispatch method is called.

        >>> print(market._generic_constraint_lhs['region'])
          set region    service  coefficient
        0   A      X     energy          1.0
        1   A      Y     energy          1.0
        2   B      X  raise_reg         -1.0

        Parameters
        ----------
        region_coefficients : pd.DataFrame

            =============  ===========================================
            Columns:       Description:
            set            the unique identifier of the constraint set \n
                           to map the lhs coefficients to, (as `str`)
            region         the region whose variables will be mapped \n
                           to the lhs, (as `str`)
            service        the service whose variables will be mapped \n
                           to the lhs, (as `str`)
            coefficient    the lhs coefficient (as `np.float64`)
            =============  ===========================================

        Raises
        ------
        RepeatedRowError
            If there is more than one row for any set, region and service combination.
        ColumnDataTypeError
            If columns are not of the required type.
        MissingColumnError
            If the column 'set', 'region', 'service' or 'coefficient' is missing.
        UnexpectedColumn
            There is a column that is not 'set', 'region', 'service' or 'coefficient'.
        ColumnValues
            If there are inf or null values in the rhs coefficient.
        """
        if self.validate_inputs:
            self._validate_generic_region_coefficients(region_coefficients)
        self._generic_constraint_lhs['region'] = region_coefficients

    def _validate_generic_region_coefficients(self, region_coefficients):
        schema = dv.DataFrameSchema(name='region_coefficients', primary_keys=['set', 'region', 'service'])
        schema.add_column(dv.SeriesSchema(name='set', data_type=str))
        schema.add_column(dv.SeriesSchema(name='region', data_type=str, allowed_values=self._market_regions))
        schema.add_column(dv.SeriesSchema(name='service', data_type=str, allowed_values=self._allowed_services))
        schema.add_column(dv.SeriesSchema(name='coefficient', data_type=np.float64, must_be_real_number=True))
        schema.validate(region_coefficients)

    def link_interconnectors_to_generic_constraints(self, interconnector_coefficients):
        """Set the lhs coefficients of generic constraints on an interconnector basis.

        Notes
        -----
        These sets also map to the set in the fcas market constraints.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['C', 'D'],
        ...     'region': ['X', 'X']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['X', 'Y'],
        ...                     unit_info=unit_info)

        Define region lhs coefficients for generic constraints. All interconnector variables are for the energy service
        so no 'service' can be specified.

        >>> interconnector_coefficients = pd.DataFrame({
        ...   'set': ['A', 'A', 'B'],
        ...   'interconnector': ['X', 'Y', 'X'],
        ...   'coefficient': [1.0, 1.0, -1.0]})

        >>> market.link_interconnectors_to_generic_constraints(interconnector_coefficients)

        Note all this does is save this information to the market object, linking to specific variable ids and
        constraint id occurs when the dispatch method is called.

        >>> print(market._generic_constraint_lhs['interconnectors'])
          set interconnector  coefficient
        0   A              X          1.0
        1   A              Y          1.0
        2   B              X         -1.0

        Parameters
        ----------
        unit_coefficients : pd.DataFrame

            =============   ==========================================
            Columns:        Description:
            set             the unique identifier of the constraint set \n
                            to map the lhs coefficients to, (as `str`)
            interconnetor   the interconnetor whose variables will be \n
                            mapped to the lhs, (as `str`)
            coefficient     the lhs coefficient (as `np.float64`)
            =============   ==========================================

        Raises
        ------
        RepeatedRowError
            If there is more than one row for any set, interconnetor and service combination.
        ColumnDataTypeError
            If columns are not of the required type.
        MissingColumnError
            If the column 'set', 'interconnetor' or 'coefficient' is missing.
        UnexpectedColumn
            There is a column that is not 'set', 'interconnetor' or 'coefficient'.
        ColumnValues
            If there are inf or null values in the rhs coefficient.
        """
        if self.validate_inputs:
            self._validate_generic_interconnector_coefficients(interconnector_coefficients)
        self._generic_constraint_lhs['interconnectors'] = interconnector_coefficients

    def _validate_generic_interconnector_coefficients(self, interconnector_coefficients):
        schema = dv.DataFrameSchema(name='interconnector_coefficients', primary_keys=['set', 'interconnector'])
        schema.add_column(dv.SeriesSchema(name='set', data_type=str))
        schema.add_column(dv.SeriesSchema(name='interconnector', data_type=str))
        schema.add_column(dv.SeriesSchema(name='coefficient', data_type=np.float64, must_be_real_number=True))
        schema.validate(interconnector_coefficients)

    def make_constraints_elastic(self, constraints_key, violation_cost):
        """Make a set of constraints elastic, so they can be violated at a predefined cost.

        If an int or float is provided as the violation_cost, then this directly sets the cost. If a pd.DataFrame
        is provided then it must contain the columns 'set' and 'cost', 'set' is used to match the cost to
        the constraints, sets in the constraints that do not appear in the pd.DataFrame will not be made
        elastic.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['C', 'D'],
        ...     'region': ['X', 'X']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['X', 'Y'],
        ...                     unit_info=unit_info)

        Define a set of generic constraints and add them to the market.

        >>> generic_constraint_parameters = pd.DataFrame({
        ...   'set': ['A', 'B'],
        ...   'type': ['>=', '<='],
        ...   'rhs': [10.0, -100.0]})

        >>> market.set_generic_constraints(generic_constraint_parameters)

        Now the market should have a set of generic constraints.

        >>> print(market._constraints_rhs_and_type['generic'])
          set  constraint_id type    rhs
        0   A              0   >=   10.0
        1   B              1   <= -100.0

        Now these constraints can be made elastic.

        >>> market.make_constraints_elastic('generic', violation_cost=1000.0)

        Now the market will contain extra decision variables to capture the cost of violating the constraint.

        >>> print(market._decision_variables['generic_deficit'])
           variable_id  lower_bound  upper_bound        type
        0            0          0.0          inf  continuous
        1            1          0.0          inf  continuous

        >>> print(market._objective_function_components['generic_deficit'])
           variable_id    cost
        0            0  1000.0
        1            1  1000.0

        These will be mapped to the constraints

        >>> print(market._lhs_coefficients['generic_deficit'])
           variable_id  constraint_id  coefficient
        0            0              0          1.0
        1            1              1         -1.0

        If a pd.DataFrame is provided then we can set cost on a constraint basis.

        >>> violation_cost = pd.DataFrame({
        ...   'set': ['A', 'B'],
        ...   'cost': [1000.0, 2000.0]})

        >>> market.make_constraints_elastic('generic', violation_cost=violation_cost)

        >>> print(market._objective_function_components['generic_deficit'])
           variable_id    cost
        0            2  1000.0
        1            3  2000.0

        Note will the variable id get incremented with every use of the method only the latest set of variables are
        used.

        Parameters
        ----------
        constraints_key : str
            The key used to reference the constraint set in the dict self.market_constraints_rhs_and_type or
            self.constraints_rhs_and_type. See the documentation for creating the constraint set to get this key.

        violation_cost : str or float or int or pd.DataFrame

        Returns
        -------
        None

        Raises
        ------
        ValueError
            If violation_cost is not str, numeric or pd.DataFrame.
        ModelBuildError
            If the constraint_key provided does not match any existing constraints.
        MissingColumnError
            If violation_cost is a pd.DataFrame and does not contain the columns set and cost.
            Or if the constraints to be made elastic do not have the set idenetifier.
        RepeatedRowError
            If violation_cost is a pd.DataFrame and has more than one row per set.
        ColumnDataTypeError
            If violation_cost is a pd.DataFrame and the column set is not str and the column
            cost is not numeric.
        """

        if constraints_key in self._market_constraints_rhs_and_type.keys():
            rhs_and_type = self._market_constraints_rhs_and_type[constraints_key].copy()
        elif constraints_key in self._constraints_rhs_and_type.keys():
            rhs_and_type = self._constraints_rhs_and_type[constraints_key].copy()
        else:
            raise check.ModelBuildError('constraints_key does not exist.')

        if isinstance(violation_cost, (int, float)) and not isinstance(violation_cost, bool):
            rhs_and_type['cost'] = violation_cost
        elif isinstance(violation_cost, pd.DataFrame):
            self._validate_violation_cost(violation_cost)
            rhs_and_type = pd.merge(rhs_and_type, violation_cost.loc[:, ['set', 'cost']], on='set')
        else:
            ValueError("Input for violation cost can only be numeric or a pd.Dataframe")

        if not rhs_and_type.empty:
            deficit_variables, lhs = elastic_constraints.create_deficit_variables(rhs_and_type, self._next_variable_id)
            self._decision_variables[constraints_key + '_deficit'] = \
                deficit_variables.loc[:, ['variable_id', 'lower_bound', 'upper_bound', 'type']]
            self._objective_function_components[constraints_key + '_deficit'] = \
                deficit_variables.loc[:, ['variable_id', 'cost']]
            self._lhs_coefficients[constraints_key + '_deficit'] = lhs
            self._next_variable_id = max(deficit_variables['variable_id']) + 1

    @staticmethod
    def _validate_violation_cost(violation_cost):
        schema = dv.DataFrameSchema(name='violation_cost', primary_keys=['set', 'cost'])
        schema.add_column(dv.SeriesSchema(name='set', data_type=str))
        schema.add_column(dv.SeriesSchema(name='cost', data_type=np.float64, must_be_real_number=True))
        schema.validate(violation_cost)

    def get_elastic_constraints_violation_degree(self, constraints_key):
        if constraints_key + '_deficit' in self._decision_variables:
            return self._decision_variables[constraints_key + '_deficit']['value'].sum()
        else:
            return 0.0

    def set_tie_break_constraints(self, cost):
        """Creates a cost that attempts to balance the energy dispatch of equally priced bids within a region.

        For each pair of bids from different generators in a region which are of the same price a constraint of the
        following form is created.

            B1 * 1/C1 - B2 * 1/C2 + D1 - D2 = 0

        Where B1 and B2 are the decision variables of each bid, C1 and C2 are the bid volumes, D1 and D2 are additional
        variables that have provided cost in the objective function. If a small cost (say 1e-6) is provided then this
        constraint balances the pro rata output of the bids.

        For AEMO documentation of this constraint
        `see AEMO doc <../../docs/pdfs/Schedule of Constraint Violation Penalty factors.pdf>` section 3 item 47.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['X', 'X']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['X'],
        ...                     unit_info=unit_info)

        Define a set of bids, in this example we have two units called A and B, with three bid bands.

        >>> volume_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [20.0, 50.0],
        ...     '2': [20.0, 30.0],
        ...     '3': [5.0, 10.0]})

        >>> market.set_unit_volume_bids(volume_bids)

        Define a set of prices for the bids.

        >>> price_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [50.0, 100.0],
        ...     '2': [100.0, 130.0],
        ...     '3': [110.0, 150.0]})

        >>> market.set_unit_price_bids(price_bids)

        Creat tie break constraints.

        >>> market.set_tie_break_constraints(1e-3)

        This should add set of constraints rhs, type and lhs coefficients

        >>> market._decision_variables['bids']
          unit capacity_band service  variable_id  lower_bound  upper_bound        type
        0    A             1  energy            0          0.0         20.0  continuous
        1    A             2  energy            1          0.0         20.0  continuous
        2    A             3  energy            2          0.0          5.0  continuous
        3    B             1  energy            3          0.0         50.0  continuous
        4    B             2  energy            4          0.0         30.0  continuous
        5    B             3  energy            5          0.0         10.0  continuous

        >>> market._constraints_rhs_and_type['tie_break']
           constraint_id type  rhs
        0              0    =  0.0

        >>> market._lhs_coefficients['tie_break']
           constraint_id  variable_id  coefficient
        0              0            1         0.05
        0              0            3        -0.02

        And a set of deficiet variables that allow the constraints to violated at the specified cost.

        >>> market._lhs_coefficients['tie_break_deficit']
           variable_id  constraint_id  coefficient
        0            6              0         -1.0
        0            7              0          1.0

        >>> market._objective_function_components['tie_break_deficit']
           variable_id   cost
        0            6  0.001
        0            7  0.001

        """

        price_bids = self._objective_function_components['bids']
        bid_decision_variables = self._decision_variables['bids']
        unit_regions = self._unit_info.loc[:, ['unit', 'region']]

        lhs, rhs = unit_constraints.tie_break_constraints(price_bids, bid_decision_variables,
                                                          unit_regions, self._next_constraint_id)

        self._lhs_coefficients['tie_break'] = lhs
        self._constraints_rhs_and_type['tie_break'] = rhs
        self._next_constraint_id = rhs['constraint_id'].max() + 1
        self.make_constraints_elastic('tie_break', violation_cost=cost)

    def dispatch(self, energy_market_ceiling_price=None, energy_market_floor_price=None, fcas_market_ceiling_price=None,
                 allow_over_constrained_dispatch_re_run=False):
        """Combines the elements of the linear program and solves to find optimal dispatch.

        If allow_over_constrained_dispatch_re_run is set to True then constraints will be relaxed when market ceiling
        or floor prices are violated.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info)

        Define a set of bids, in this example we have two units called A and B, with three bid bands.

        >>> volume_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [20.0, 50.0],
        ...     '2': [20.0, 30.0],
        ...     '3': [5.0, 10.0]})

        Create energy unit bid decision variables.

        >>> market.set_unit_volume_bids(volume_bids)

        Define a set of prices for the bids.

        >>> price_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [50.0, 100.0],
        ...     '2': [100.0, 130.0],
        ...     '3': [100.0, 150.0]})

        Create the objective function components corresponding to the the energy bids.

        >>> market.set_unit_price_bids(price_bids)

        Define a demand level in each region.

        >>> demand = pd.DataFrame({
        ...     'region': ['NSW'],
        ...     'demand': [100.0]})

        Create unit capacity based constraints.

        >>> market.set_demand_constraints(demand)

        Call the dispatch method.

        >>> market.dispatch()

        Now the market dispatch can be retrieved.

        >>> print(market.get_unit_dispatch())
          unit service  dispatch
        0    A  energy      45.0
        1    B  energy      55.0

        And the market prices can be retrieved.

        >>> print(market.get_energy_prices())
          region  price
        0    NSW  130.0

        Returns
        -------
        None

        Raises
        ------
            ModelBuildError
                If a model build process is incomplete, i.e. there are energy bids but not energy demand set.
        """
        if allow_over_constrained_dispatch_re_run:
            if (energy_market_ceiling_price is None or energy_market_floor_price is None or
                    fcas_market_ceiling_price is None):
                raise ValueError("""If allow_over_constrained_dispatch_re_run is set to True then values must \n
                                    be provided for energy_market_ceiling_price, energy_market_floor_price, and \n
                                    fcas_market_ceiling_price.""")

        # Create a data frame containing all fully defined components of the constraint matrix lhs. If there are none
        # then just create a place holder empty pd.DataFrame.
        if len(self._lhs_coefficients.values()) > 0:
            constraints_lhs = pd.concat(list(self._lhs_coefficients.values()))
        else:
            constraints_lhs = pd.DataFrame()

        # Get a pd.DataFrame mapping the generic constraint sets to their constraint ids.
        generic_constraint_ids = solver_interface.create_mapping_of_generic_constraint_sets_to_constraint_ids(
            self._constraints_rhs_and_type, self._market_constraints_rhs_and_type)

        # If there are any generic constraints create their lhs definitions.
        if generic_constraint_ids is not None:
            generic_lhs = []
            # If units have been added to the generic lhs then find the relevant variable ids and map them to the
            # constraint.
            if 'unit' in self._generic_constraint_lhs and 'bids' in self._variable_to_constraint_map['unit_level']:
                generic_constraint_units = self._generic_constraint_lhs['unit']
                unit_bids_to_constraint_map = self._variable_to_constraint_map['unit_level']['bids']
                unit_lhs = solver_interface.create_unit_level_generic_constraint_lhs(generic_constraint_units,
                                                                                     generic_constraint_ids,
                                                                                     unit_bids_to_constraint_map)
                generic_lhs.append(unit_lhs)
            # If regions have been added to the generic lhs then find the relevant variable ids and map them to the
            # constraint.
            if 'region' in self._generic_constraint_lhs and 'bids' in self._variable_to_constraint_map['regional']:
                generic_constraint_region = self._generic_constraint_lhs['region']
                unit_bids_to_constraint_map = self._variable_to_constraint_map['regional']['bids']
                regional_lhs = solver_interface.create_region_level_generic_constraint_lhs(generic_constraint_region,
                                                                                           generic_constraint_ids,
                                                                                           unit_bids_to_constraint_map)
                generic_lhs.append(regional_lhs)
            # If interconnectors have been added to the generic lhs then find the relevant variable ids and map them
            # to the constraint.
            if 'interconnectors' in self._generic_constraint_lhs and 'interconnectors' in self._decision_variables:
                generic_constraint_interconnectors = self._generic_constraint_lhs['interconnectors']
                interconnector_bids_to_constraint_map = self._decision_variables['interconnectors']
                interconnector_lhs = solver_interface.create_interconnector_generic_constraint_lhs(
                    generic_constraint_interconnectors, generic_constraint_ids, interconnector_bids_to_constraint_map)
                generic_lhs.append(interconnector_lhs)
            # Add the generic lhs definitions the cumulative lhs pd.DataFrame.
            constraints_lhs = pd.concat([constraints_lhs] + generic_lhs)

        # If there are constraints that have been defined on a regional basis then create the constraints lhs
        # definition by mapping to all the variables that have been defined for the corresponding region and service.
        if len(self._constraint_to_variable_map['regional']) > 0:
            constraints = pd.concat(list(self._constraint_to_variable_map['regional'].values()))
            decision_variables = pd.concat(list(self._variable_to_constraint_map['regional'].values()))
            regional_constraints_lhs = solver_interface.create_lhs(constraints, decision_variables,
                                                                   ['region', 'service'])
            # Add the lhs definitions the cumulative lhs pd.DataFrame.
            constraints_lhs = pd.concat([constraints_lhs, regional_constraints_lhs])

        # If there are constraints that have been defined on a unit basis then create the constraints lhs
        # definition by mapping to all the variables that have been defined for the corresponding unit and service.
        if len(self._constraint_to_variable_map['unit_level']) > 0:
            constraints = pd.concat(list(self._constraint_to_variable_map['unit_level'].values()))
            decision_variables = pd.concat(list(self._variable_to_constraint_map['unit_level'].values()))
            unit_constraints_lhs = solver_interface.create_lhs(constraints, decision_variables, ['unit', 'service'])
            # Add the lhs definitions the cumulative lhs pd.DataFrame.
            constraints_lhs = pd.concat([constraints_lhs, unit_constraints_lhs])

        # Create the interface to the solver.
        si = solver_interface.InterfaceToSolver(self.solver_name)
        if self._decision_variables:
            # Combine dictionary of pd.DataFrames into a single pd.DataFrame for processing by the interface.
            variable_definitions = pd.concat(self._decision_variables)
            si.add_variables(variable_definitions)
        else:
            raise check.ModelBuildError('The market could not be dispatch because no variables have been created')

        # If Costs have been defined for bids or constraints then add an objective function.
        if self._objective_function_components:
            # Combine components of objective function into a single pd.DataFrame
            objective_function_definition = pd.concat(self._objective_function_components)
            si.add_objective_function(objective_function_definition)

        # Collect all constraint rhs and type definitions into a single pd.DataFrame.
        constraints_rhs_and_type = []
        if self._constraints_rhs_and_type:
            constraints_rhs_and_type.append(pd.concat(self._constraints_rhs_and_type))
        if self._market_constraints_rhs_and_type:
            constraints_rhs_and_type.append(pd.concat(self._market_constraints_rhs_and_type))
        if self._constraints_dynamic_rhs_and_type:
            constraints_dynamic_rhs_and_type = pd.concat(self._constraints_dynamic_rhs_and_type)
            # Create the rhs for the dynamic constraints.
            constraints_dynamic_rhs_and_type['rhs'] = constraints_dynamic_rhs_and_type. \
                apply(lambda x: si.variables[x['rhs_variable_id']], axis=1)
            constraints_rhs_and_type.append(constraints_dynamic_rhs_and_type)

        if len(constraints_rhs_and_type) > 0:
            constraints_rhs_and_type = pd.concat(constraints_rhs_and_type)
            si.add_constraints(constraints_lhs, constraints_rhs_and_type)

        # If interconnectors with losses are being used, create special ordered sets for modelling losses.
        if 'interpolation_weights' in self._decision_variables:
            special_ordered_sets = self._decision_variables['interpolation_weights']
            si.add_sos_type_2(special_ordered_sets, sos_id_columns=['interconnector', 'link'],
                              position_column='loss_segment')

        if 'interconnectors' in self._decision_variables:
            special_ordered_sets = self._decision_variables['interconnectors']
            special_ordered_sets = special_ordered_sets[
                special_ordered_sets['interconnector'] != special_ordered_sets['link']]
            if not special_ordered_sets.empty:
                special_ordered_sets = special_ordered_sets.rename(columns={'interconnector': 'sos_id'})
                si.add_sos_type_1(special_ordered_sets)

        si.optimize()

        # Find the slack in constraints.
        if self._constraints_rhs_and_type:
            for constraint_group in self._constraints_rhs_and_type:
                self._constraints_rhs_and_type[constraint_group]['slack'] = \
                    si.get_slack_in_constraints(self._constraints_rhs_and_type[constraint_group])
        if self._market_constraints_rhs_and_type:
            for constraint_group in self._market_constraints_rhs_and_type:
                self._market_constraints_rhs_and_type[constraint_group]['slack'] = \
                    si.get_slack_in_constraints(self._market_constraints_rhs_and_type[constraint_group])
        if self._constraints_dynamic_rhs_and_type:
            for constraint_group in self._constraints_dynamic_rhs_and_type:
                self._constraints_dynamic_rhs_and_type[constraint_group]['slack'] = \
                    si.get_slack_in_constraints(self._constraints_dynamic_rhs_and_type[constraint_group])

        # Get decision variable optimal values
        for var_group in self._decision_variables:
            self._decision_variables[var_group]['value'] = \
                si.get_optimal_values_of_decision_variables(self._decision_variables[var_group])

        # Models with interconnectors use binary variables, the model needs to be linearised to allow for shadow prices
        # to be accessed and used to price constraints.
        if 'interconnector_losses' in self._decision_variables:
            si = self._get_linear_model(si)
        si.linear_mip_model.optimize()

        for var_group in self._decision_variables:
            self._decision_variables[var_group]['value_lin'] = \
                si.get_optimal_values_of_decision_variables_lin(self._decision_variables[var_group])

        # If there are market constraints then calculate their associated prices.
        if self._market_constraints_rhs_and_type:
            for constraint_group in self._market_constraints_rhs_and_type:
                constraints_to_price = list(self._market_constraints_rhs_and_type[constraint_group]['constraint_id'])
                prices = si.price_constraints(constraints_to_price)
                self._market_constraints_rhs_and_type[constraint_group]['price'] = \
                    self._market_constraints_rhs_and_type[constraint_group]['constraint_id'].map(prices)

        if allow_over_constrained_dispatch_re_run:
            fcas_ceiling_price_violated = False
            if 'fcas' in self._market_constraints_rhs_and_type:
                if self._market_constraints_rhs_and_type['fcas']['price'].max() >= fcas_market_ceiling_price:
                    fcas_ceiling_price_violated = True

            energy_ceiling_price_violated = False
            if 'demand' in self._market_constraints_rhs_and_type:
                if self._market_constraints_rhs_and_type['demand']['price'].max() >= energy_market_ceiling_price:
                    energy_ceiling_price_violated = True

            energy_floor_price_violated = False
            if 'demand' in self._market_constraints_rhs_and_type:
                if self._market_constraints_rhs_and_type['demand']['price'].min() <= energy_market_floor_price:
                    energy_floor_price_violated = True

            deficit_variables = []
            lhs_deficit_variables = []

            generic_cons_violated = False
            if 'generic_deficit' in self._decision_variables:
                if (self._decision_variables['generic_deficit']['value'].max() > 0.0001 or
                        self._decision_variables['generic_deficit']['value'].min() < -0.0001):
                    generic_cons_violated = True
                    deficit_variables.append(self._decision_variables['generic_deficit'].copy())
                    lhs_deficit_variables.append(self._lhs_coefficients['generic_deficit'])

            fcas_cons_violated = False
            if 'fcas_deficit' in self._decision_variables:
                if (self._decision_variables['fcas_deficit']['value'].max() > 0.0001 or
                        self._decision_variables['fcas_deficit']['value'].min() < -0.0001):
                    fcas_cons_violated = True
                    deficit_variables.append(self._decision_variables['fcas_deficit'].copy())
                    lhs_deficit_variables.append(self._lhs_coefficients['fcas_deficit'])

            if ((fcas_ceiling_price_violated or energy_ceiling_price_violated or energy_floor_price_violated) and
                    (generic_cons_violated or fcas_cons_violated)):
                variables = pd.concat(deficit_variables)
                active_violation_variables = variables[(variables['value'] > 0.0) | (variables['value'] < -0.0)]
                lhs = pd.concat(lhs_deficit_variables)
                variables_and_cons = pd.merge(active_violation_variables, lhs, on='variable_id')
                variables_and_cons['adjuster'] = (variables_and_cons['value'] + 0.01) * \
                                                 variables_and_cons['coefficient'] * -1
                variables_and_cons.apply(lambda x: si.update_rhs(x['constraint_id'], x['adjuster']), axis=1)
                si.linear_mip_model.optimize()

                # If there are market constraints then calculate their associated prices.
                if self._market_constraints_rhs_and_type:
                    for constraint_group in self._market_constraints_rhs_and_type:
                        constraints_to_price = list(
                            self._market_constraints_rhs_and_type[constraint_group]['constraint_id'])
                        prices = si.price_constraints(constraints_to_price)
                        self._market_constraints_rhs_and_type[constraint_group]['price'] = \
                            self._market_constraints_rhs_and_type[constraint_group]['constraint_id'].map(prices)

        self.objective_value = si.mip_model.objective_value

    def _get_linear_model(self, si):
        self._remove_unused_interpolation_weights(si)
        self._disable_unused_link_pair(si)
        return si

    def _disable_unused_link_pair(self, si):
        inter_vars = self._decision_variables['interconnectors']
        inter_vars = inter_vars[inter_vars['interconnector'] != inter_vars['link']]
        inter_vars_unused = inter_vars[inter_vars['value'] == 0.0]
        si.disable_variables(inter_vars_unused)

    def _remove_unused_interpolation_weights(self, si):
        vars = pd.merge(self._decision_variables['interconnectors'].loc[:, ['interconnector', 'link', 'value']],
                        self._decision_variables['interpolation_weights'].loc[:, ['interconnector', 'link',
                                                                                  'break_point', 'variable_id']],
                        on=['interconnector', 'link'])
        vars['distance'] = (vars['value'] - vars['break_point']).abs()

        def not_closest_three(df):
            df = df.sort_values('distance')
            df = df.iloc[3:, :]
            return df

        vars_to_remove = vars.groupby(['interconnector', 'link'], as_index=False).apply(not_closest_three)
        si.disable_variables(vars_to_remove.loc[:, ['variable_id']])

    def get_constraint_set_names(self):
        return list(self._market_constraints_rhs_and_type.keys()) + list(self._constraints_rhs_and_type.keys())

    def get_unit_dispatch(self):
        """Retrieves the energy dispatch for each unit.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info)

        Define a set of bids, in this example we have two units called A and B, with three bid bands.

        >>> volume_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [20.0, 50.0],
        ...     '2': [20.0, 30.0],
        ...     '3': [5.0, 10.0]})

        Create energy unit bid decision variables.

        >>> market.set_unit_volume_bids(volume_bids)

        Define a set of prices for the bids.

        >>> price_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [50.0, 100.0],
        ...     '2': [100.0, 130.0],
        ...     '3': [100.0, 150.0]})

        Create the objective function components corresponding to the the energy bids.

        >>> market.set_unit_price_bids(price_bids)

        Define a demand level in each region.

        >>> demand = pd.DataFrame({
        ...     'region': ['NSW'],
        ...     'demand': [100.0]})

        Create unit capacity based constraints.

        >>> market.set_demand_constraints(demand)

        Call the dispatch method.

        >>> market.dispatch()

        Now the market dispatch can be retrieved.

        >>> print(market.get_unit_dispatch())
          unit service  dispatch
        0    A  energy      45.0
        1    B  energy      55.0

        Returns
        -------
        pd.DataFrame

        Raises
        ------
            ModelBuildError
                If a model build process is incomplete, i.e. there are energy bids but not energy demand set.
        """
        dispatch = self._decision_variables['bids'].loc[:, ['unit', 'service', 'value']]
        dispatch.columns = ['unit', 'service', 'dispatch']
        return dispatch.groupby(['unit', 'service'], as_index=False).sum()

    def get_energy_prices(self):
        """Retrieves the energy price in each market region.

        Energy prices are the shadow prices of the demand constraint in each market region.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW'],
        ...                     unit_info=unit_info)

        Define a set of bids, in this example we have two units called A and B, with three bid bands.

        >>> volume_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [20.0, 50.0],
        ...     '2': [20.0, 30.0],
        ...     '3': [5.0, 10.0]})

        Create energy unit bid decision variables.

        >>> market.set_unit_volume_bids(volume_bids)

        Define a set of prices for the bids.

        >>> price_bids = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     '1': [50.0, 100.0],
        ...     '2': [100.0, 130.0],
        ...     '3': [100.0, 150.0]})

        Create the objective function components corresponding to the the energy bids.

        >>> market.set_unit_price_bids(price_bids)

        Define a demand level in each region.

        >>> demand = pd.DataFrame({
        ...     'region': ['NSW'],
        ...     'demand': [100.0]})

        Create unit capacity based constraints.

        >>> market.set_demand_constraints(demand)

        Call the dispatch method.

        >>> market.dispatch()

        Now the market prices can be retrieved.

        >>> print(market.get_energy_prices())
          region  price
        0    NSW  130.0

        Returns
        -------
        pd.DateFrame

        Raises
        ------
            ModelBuildError
                If a model build process is incomplete, i.e. there are energy bids but not energy demand set.
        """
        prices = self._market_constraints_rhs_and_type['demand'].loc[:, ['region', 'price']]
        return prices

    def get_fcas_prices(self):
        """Retrives the price associated with each set of FCAS requirement constraints.

        Returns
        -------
        pd.DateFrame
        """
        prices = pd.merge(
            self._constraint_to_variable_map['regional']['fcas'].loc[:, ['service', 'region', 'constraint_id']],
            self._market_constraints_rhs_and_type['fcas'].loc[:, ['set', 'price', 'constraint_id']], on='constraint_id')
        prices = prices.groupby(['region', 'service'], as_index=False).aggregate({'price': 'sum'})
        return prices

    def get_interconnector_flows(self):
        """Retrieves the  flows for each interconnector.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW', 'VIC'],
        ...                     unit_info=unit_info)

        Define a set of bids, in this example we have just one unit that can provide 100 MW in NSW.

        >>> volume_bids = pd.DataFrame({
        ...     'unit': ['A'],
        ...     '1': [100.0]})

        Create energy unit bid decision variables.

        >>> market.set_unit_volume_bids(volume_bids)

        Define a set of prices for the bids.

        >>> price_bids = pd.DataFrame({
        ...     'unit': ['A'],
        ...     '1': [80.0]})

        Create the objective function components corresponding to the the energy bids.

        >>> market.set_unit_price_bids(price_bids)

        Define a demand level in each region, no power is required in NSW and 90.0 MW is required in VIC.

        >>> demand = pd.DataFrame({
        ...     'region': ['NSW', 'VIC'],
        ...     'demand': [0.0, 90.0]})

        Create unit capacity based constraints.

        >>> market.set_demand_constraints(demand)

        Define a an interconnector between NSW and VIC so generator can A can be used to meet demand in VIC.

        >>> interconnector = pd.DataFrame({
        ...     'interconnector': ['inter_one'],
        ...     'to_region': ['VIC'],
        ...     'from_region': ['NSW'],
        ...     'max': [100.0],
        ...     'min': [-100.0]})

        Create the interconnector.

        >>> market.set_interconnectors(interconnector)

        Call the dispatch method.

        >>> market.dispatch()

        Now the market dispatch can be retrieved.

        >>> print(market.get_unit_dispatch())
          unit service  dispatch
        0    A  energy      90.0

        And the interconnector flows can be retrieved.

        >>> print(market.get_interconnector_flows())
          interconnector       link  flow
        0      inter_one  inter_one  90.0

        Returns
        -------
        pd.DataFrame

        """
        flow = self._decision_variables['interconnectors'].loc[:, ['interconnector', 'link', 'value']]
        flow.columns = ['interconnector', 'link', 'flow']

        if 'interconnector_losses' in self._decision_variables:
            losses = self._decision_variables['interconnector_losses'].loc[:, ['interconnector', 'link', 'value']]
            losses.columns = ['interconnector', 'link', 'losses']
            flow = pd.merge(flow, losses, 'left', on=['interconnector', 'link'])

        return flow.reset_index(drop=True)

    def get_region_dispatch_summary(self):
        """Calculates a dispatch summary at the regional level.

        Examples
        --------
        Define the unit information data set needed to initialise the market.

        >>> unit_info = pd.DataFrame({
        ...     'unit': ['A', 'B'],
        ...     'region': ['NSW', 'NSW']})

        Initialise the market instance.

        >>> market = SpotMarket(market_regions=['NSW', 'VIC'],
        ...                     unit_info=unit_info)

        Define a set of bids, in this example we have just one unit that can provide 100 MW in NSW.

        >>> volume_bids = pd.DataFrame({
        ...     'unit': ['A'],
        ...     '1': [100.0]})

        Create energy unit bid decision variables.

        >>> market.set_unit_volume_bids(volume_bids)

        Define a set of prices for the bids.

        >>> price_bids = pd.DataFrame({
        ...     'unit': ['A'],
        ...     '1': [80.0]})

        Create the objective function components corresponding to the the energy bids.

        >>> market.set_unit_price_bids(price_bids)

        Define a demand level in each region, no power is required in NSW and 90.0 MW is required in VIC.

        >>> demand = pd.DataFrame({
        ...     'region': ['NSW', 'VIC'],
        ...     'demand': [0.0, 90.0]})

        Create unit capacity based constraints.

        >>> market.set_demand_constraints(demand)

        Define a an interconnector between NSW and VIC so generator can A can be used to meet demand in VIC.

        >>> interconnector = pd.DataFrame({
        ...     'interconnector': ['inter_one'],
        ...     'to_region': ['VIC'],
        ...     'from_region': ['NSW'],
        ...     'max': [100.0],
        ...     'min': [-100.0]})

        Create the interconnector.

        >>> market.set_interconnectors(interconnector)

        Define the interconnector loss function. In this case losses are always 5 % of line flow.

        >>> def constant_losses(flow=None):
        ...     return abs(flow) * 0.05

        Define the function on a per interconnector basis. Also details how the losses should be proportioned to the
        connected regions.

        >>> loss_functions = pd.DataFrame({
        ...    'interconnector': ['inter_one'],
        ...    'from_region_loss_share': [0.5],  # losses are shared equally.
        ...    'loss_function': [constant_losses]})

        Define the points to linearly interpolate the loss function between. In this example the loss function is
        linear so only three points are needed, but if a non linear loss function was used then more points would
        result in a better approximation.

        >>> interpolation_break_points = pd.DataFrame({
        ...    'interconnector': ['inter_one', 'inter_one', 'inter_one'],
        ...    'loss_segment': [1, 2, 3],
        ...    'break_point': [-120.0, 0.0, 100]})

        >>> market.set_interconnector_losses(loss_functions, interpolation_break_points)

        Call the dispatch method.

        >>> market.dispatch()

        Now the region dispatch summary can be retreived.

        >>> print(market.get_region_dispatch_summary())
          region   dispatch     inflow  transmission_losses  interconnector_losses
        0    NSW  94.615385 -92.307692                  0.0               2.307692
        1    VIC   0.000000  92.307692                  0.0               2.307692

        Returns
        -------
        pd.DataFrame

            =====================    =================================
            Columns:                 Description:
            region                   unique identifier of a market \n
                                     region, required (as `str`)
            dispatch                 the net dispatch of units inside \n
                                     a region i.e. generators dispatch \n
                                     minus load dispatch, in MW. (as `np.float64`)
            inflow                   the net inflow from interconnectors, \n
                                     not including losses, in MW \n
                                     (as `np.float64`)
            interconnector_losses    interconnector losses attributed \n
                                     to region, in MW, (as `np.float64`)
            =====================    =================================
        """
        dispatch_summary = self._get_net_unit_dispatch_by_region()
        if self._interconnectors_in_market():
            interconnector_inflow = self._get_interconnector_inflow_by_region()
            dispatch_summary = pd.merge(dispatch_summary, interconnector_inflow, how='outer', on='region')
            dispatch_summary = dispatch_summary.fillna(0.0)
            transmission_losses = self._get_transmission_losses()
            dispatch_summary = pd.merge(dispatch_summary, transmission_losses, on='region')
        if self._interconnectors_have_losses():
            interconnector_losses = self._get_interconnector_losses_by_region()
            dispatch_summary = pd.merge(dispatch_summary, interconnector_losses, on='region')
        return dispatch_summary

    def _get_net_unit_dispatch_by_region(self):

        unit_dispatch = self.get_unit_dispatch()
        unit_dispatch = unit_dispatch[unit_dispatch['service'] == 'energy']
        unit_dispatch_types = self._unit_info.loc[:, ['unit', 'region', 'dispatch_type']]
        unit_dispatch = pd.merge(unit_dispatch, unit_dispatch_types, on='unit')

        def make_load_dispatch_negative(dispatch_type, dispatch):
            if dispatch_type == 'load':
                dispatch = -1 * dispatch
            return dispatch

        unit_dispatch['dispatch'] = \
            unit_dispatch.apply(lambda x: make_load_dispatch_negative(x['dispatch_type'], x['dispatch']), axis=1)

        unit_dispatch = unit_dispatch.groupby('region', as_index=False).aggregate({'dispatch': 'sum'})
        return unit_dispatch

    def _interconnectors_in_market(self):
        return self._interconnector_directions is not None

    def _get_interconnector_inflow_by_region(self):

        def calc_inflow_by_interconnector(interconnector_direction_coefficients, interconnector_flows):
            inflow = pd.merge(interconnector_direction_coefficients, interconnector_flows,
                              on=['interconnector', 'link'])
            inflow['inflow'] = inflow['flow'] * inflow['direction_coefficient']
            return inflow

        def calc_inflow_by_region(inflow):
            inflow = inflow.groupby('region', as_index=False).aggregate({'inflow': 'sum'})
            return inflow

        interconnector_flows = self.get_interconnector_flows()
        interconnector_direction_coefficients = self._get_interconnector_inflow_coefficients()
        inflow = calc_inflow_by_interconnector(interconnector_direction_coefficients, interconnector_flows)
        inflow = calc_inflow_by_region(inflow)

        return inflow

    def _get_interconnector_inflow_coefficients(self):

        def define_positive_inflows(interconnectors):
            inflow_direction = interconnectors.loc[:, ['interconnector', 'link', 'to_region']]
            inflow_direction['direction_coefficient'] = 1.0
            inflow_direction.columns = ['interconnector', 'link', 'region', 'direction_coefficient']
            return inflow_direction

        def define_negative_inflows(interconnectors):
            outflow_direction = interconnectors.loc[:, ['interconnector', 'link', 'from_region']]
            outflow_direction['direction_coefficient'] = -1.0
            outflow_direction.columns = ['interconnector', 'link', 'region', 'direction_coefficient']
            return outflow_direction

        positive = define_positive_inflows(self._interconnector_directions)
        negative = define_negative_inflows(self._interconnector_directions)

        return pd.concat([positive, negative])

    def _interconnectors_have_losses(self):
        return self._interconnector_loss_shares is not None

    def _get_interconnector_losses_by_region(self):
        from_region_loss_shares = self._get_from_region_loss_shares()
        to_region_loss_shares = self._get_to_region_loss_shares()
        loss_shares = pd.concat([from_region_loss_shares, to_region_loss_shares])
        losses = self.get_interconnector_flows().loc[:, ['interconnector', 'link', 'losses']]
        losses = pd.merge(losses, loss_shares, on=['interconnector', 'link'])
        losses['interconnector_losses'] = losses['losses'] * losses['loss_share']
        losses = losses.groupby('region', as_index=False).aggregate({'interconnector_losses': 'sum'})
        return losses

    def _get_from_region_loss_shares(self):
        from_region_loss_share = self._get_loss_shares('from_region')
        from_region_loss_share = from_region_loss_share.rename(columns={'from_region_loss_share': 'loss_share'})
        return from_region_loss_share

    def _get_to_region_loss_shares(self):
        to_region_loss_share = self._get_loss_shares('to_region')
        to_region_loss_share['loss_share'] = 1 - to_region_loss_share['from_region_loss_share']
        to_region_loss_share = to_region_loss_share.drop('from_region_loss_share', axis=1)
        return to_region_loss_share

    def _get_loss_shares(self, region_type):
        from_region_loss_share = self._interconnector_loss_shares
        regions = self._interconnector_directions.loc[:, ['interconnector', 'link', region_type]]
        regions = regions.rename(columns={region_type: 'region'})
        from_region_loss_share = pd.merge(from_region_loss_share, regions, on=['interconnector', 'link'])
        from_region_loss_share = from_region_loss_share.loc[:, ['interconnector', 'link', 'region',
                                                                'from_region_loss_share']]
        return from_region_loss_share

    def _get_transmission_losses(self):
        interconnector_directions = self._interconnector_directions
        loss_factors = hf.stack_columns(interconnector_directions, ['interconnector', 'link'],
                                        ['from_region_loss_factor', 'to_region_loss_factor'], 'direction',
                                        'loss_factor')
        interconnector_directions = hf.stack_columns(interconnector_directions, ['interconnector', 'link'],
                                                     ['to_region', 'from_region'], 'direction', 'region')
        loss_factors['direction'] = loss_factors['direction'].apply(lambda x: x.replace('_loss_factor', ''))
        loss_factors = pd.merge(loss_factors, interconnector_directions, on=['interconnector', 'link', 'direction'])
        flows_and_losses = self.get_interconnector_flows()
        flows_and_losses = pd.merge(flows_and_losses, loss_factors, on=['interconnector', 'link'])

        def calc_losses(direction, flow, loss_factor):
            if (direction == 'to_region' and flow >= 0.0) or (direction == 'from_region' and flow <= 0.0):
                losses = flow * (1 - loss_factor)
            elif (direction == 'to_region' and flow < 0.0) or (direction == 'from_region' and flow > 0.0):
                losses = abs(flow) - (abs(flow) / loss_factor)
            return losses

        flows_and_losses['transmission_losses'] = \
            flows_and_losses.apply(lambda x: calc_losses(x['direction'], x['flow'], x['loss_factor']), axis=1)
        flows_and_losses = flows_and_losses.groupby('region', as_index=False).aggregate({'transmission_losses': 'sum'})
        return flows_and_losses

    def get_fcas_availability(self):
        """Get the availability of fcas service on a unit level, after constraints.

        Examples
        --------
        Volume of each bid.

        >>> volume_bids = pd.DataFrame({
        ...   'unit': ['A', 'A', 'B', 'B', 'B'],
        ...   'service': ['energy', 'raise_6s', 'energy',
        ...               'raise_6s', 'raise_reg'],
        ...   '1': [100.0, 10.0, 110.0, 15.0, 15.0]})

        Price of each bid.

        >>> price_bids = pd.DataFrame({
        ...   'unit': ['A', 'A', 'B', 'B', 'B'],
        ...   'service': ['energy', 'raise_6s', 'energy',
        ...               'raise_6s', 'raise_reg'],
        ...   '1': [50.0, 35.0, 60.0, 20.0, 30.0]})

        Participant defined operational constraints on FCAS enablement.

        >>> fcas_trapeziums = pd.DataFrame({
        ...   'unit': ['B', 'B', 'A'],
        ...   'service': ['raise_reg', 'raise_6s', 'raise_6s'],
        ...   'max_availability': [15.0, 15.0, 10.0],
        ...   'enablement_min': [50.0, 50.0, 70.0],
        ...   'low_break_point': [65.0, 65.0, 80.0],
        ...   'high_break_point': [95.0, 95.0, 100.0],
        ...   'enablement_max': [110.0, 110.0, 110.0]})

        Unit locations.

        >>> unit_info = pd.DataFrame({
        ...   'unit': ['A', 'B'],
        ...   'region': ['NSW', 'NSW']})

        The demand in the regions being dispatched.

        >>> demand = pd.DataFrame({
        ...   'region': ['NSW'],
        ...   'demand': [195.0]})

        FCAS requirement in the regions being dispatched.

        >>> fcas_requirements = pd.DataFrame({
        ...   'set': ['nsw_regulation_requirement',
        ...           'nsw_raise_6s_requirement'],
        ...   'region': ['NSW', 'NSW'],
        ...   'service': ['raise_reg', 'raise_6s'],
        ...   'volume': [10.0, 10.0]})

        Create the market model with unit service bids.

        >>> market = SpotMarket(unit_info=unit_info,
        ...                     market_regions=['NSW'])
        >>> market.set_unit_volume_bids(volume_bids)
        >>> market.set_unit_price_bids(price_bids)

        Create constraints that enforce the top of the FCAS trapezium.

        >>> fcas_availability = fcas_trapeziums.loc[:, ['unit', 'service', 'max_availability']]
        >>> market.set_fcas_max_availability(fcas_availability)

        Create constraints the enforce the lower and upper slope of the FCAS regulation service trapeziums.

        >>> regulation_trapeziums = fcas_trapeziums[fcas_trapeziums['service'] == 'raise_reg']
        >>> market.set_energy_and_regulation_capacity_constraints(regulation_trapeziums)

        Create constraints that enforce the lower and upper slope of the FCAS contingency
        trapezium. These constrains also scale slopes of the trapezium to ensure the
        co-dispatch of contingency and regulation services is technically feasible.

        >>> contingency_trapeziums = fcas_trapeziums[fcas_trapeziums['service'] == 'raise_6s']
        >>> market.set_joint_capacity_constraints(contingency_trapeziums)

        Set the demand for energy.

        >>> market.set_demand_constraints(demand)

        Set the required volume of FCAS services.

        >>> market.set_fcas_requirements_constraints(fcas_requirements)

        Calculate dispatch and pricing

        >>> market.dispatch()

        Return the total dispatch of each unit in MW.

        >>> print(market.get_unit_dispatch())
          unit    service  dispatch
        0    A     energy     100.0
        1    A   raise_6s       5.0
        2    B     energy      95.0
        3    B   raise_6s       5.0
        4    B  raise_reg      10.0

        Return the constrained availability of each units fcas service.

        >>> print(market.get_fcas_availability())
          unit    service  availability
        0    A   raise_6s          10.0
        1    B   raise_6s           5.0
        2    B  raise_reg          10.0

        Returns
        -------

        """
        fcas_variable_slack = []
        for constraint_type in ['fcas_max_availability', 'joint_ramping_raise_reg', 'joint_ramping_lower_reg',
                                'joint_capacity', 'energy_and_regulation_capacity']:
            if constraint_type in self._constraints_rhs_and_type.keys():
                service_coefficients = self._constraint_to_variable_map['unit_level'][constraint_type]
                service_coefficients = service_coefficients.loc[:, ['constraint_id', 'unit', 'service', 'coefficient']]
                constraint_slack = self._constraints_rhs_and_type[constraint_type].loc[:, ['constraint_id', 'slack',
                                                                                           'type']]
                slack_temp = pd.merge(service_coefficients, constraint_slack, on='constraint_id')
                fcas_variable_slack.append(slack_temp)

        fcas_variable_slack = pd.concat(fcas_variable_slack)
        fcas_variable_slack['service_slack'] = \
            np.where(((fcas_variable_slack['coefficient'] < 0.0) & (fcas_variable_slack['type'] == '<=')) |
                     ((fcas_variable_slack['coefficient'] > 0.0) & (fcas_variable_slack['type'] == '>=')) |
                     ((fcas_variable_slack['coefficient'] < 0.00001) & (fcas_variable_slack['coefficient'] > -0.00001)),
                     np.Inf, fcas_variable_slack['slack'].abs() / fcas_variable_slack['coefficient'].abs())
        fcas_variable_slack = \
            fcas_variable_slack.groupby(['unit', 'service'], as_index=False).aggregate({'service_slack': 'min'})
        fcas_variable_slack = fcas_variable_slack[fcas_variable_slack['service'] != 'energy']

        dispatch_levels = self.get_unit_dispatch()

        fcas_availability = pd.merge(fcas_variable_slack, dispatch_levels, on=['unit', 'service'])

        fcas_availability['availability'] = fcas_availability['dispatch'] + fcas_availability['service_slack']
        return fcas_availability.loc[:, ['unit', 'service', 'availability']]


class ModelBuildError(Exception):
    """Raise for building model components in wrong order."""


class MissingTable(Exception):
    """Raise for trying to access missing table."""

================
File: nempy\time_sequential.py
================
import pandas as pd


def construct_ramp_rate_parameters(last_interval_dispatch, ramp_rates):
    """Combine dispatch and ramp rates into the ramp rate inputs compatible with the SpotMarket class.

    Examples
    -------

    >>> last_interval_dispatch = pd.DataFrame({
    ... 'unit': ['A', 'A', 'B'],
    ... 'service': ['energy', 'raise_reg', 'energy'],
    ... 'dispatch': [45.0, 50.0, 88.0]})

    >>> ramp_rates = pd.DataFrame({
    ... 'unit': ['A', 'B', 'C'],
    ... 'ramp_up_rate': [600.0, 1200.0, 700.0],
    ... 'ramp_down_rate': [600.0, 1200.0, 700.0]})

    >>> construct_ramp_rate_parameters(last_interval_dispatch,
    ...                                ramp_rates)
      unit  initial_output  ramp_up_rate  ramp_down_rate
    0    A            45.0         600.0           600.0
    1    B            88.0        1200.0          1200.0
    2    C             0.0         700.0           700.0

    Parameters
    ----------
    last_interval_dispatch : pd.DataFrame

        ========  ================================================
        Columns:  Description:
        unit      unique identifier of a dispatch unit (as `str`)
        service   the service being provided, optional, \n
                  default 'energy', (as `str`)
        dispatch  the dispatch target from the previous dispatch \n
                  interval, in MW, (as `np.float64`)
        ========  ================================================

    ramp_rates : pd.DataFrame

        ================  ========================================
        Columns:          Description:
        unit              unique identifier for units, (as `str`) \n
        ramp_up_rate      the ramp up rate, in MW/h, \n
                          (as `np.float64`)
        ramp_down_rate    the ramp down rate, in MW/h, \n
                          (as `np.float64`)
        ================  ========================================

    Returns
    -------
    pd.DataFrame

        ================  ========================================
        Columns:          Description:
        unit              unique identifier for units, (as `str`) \n
        initial_output    the output/consumption of the unit at \n
                          the start of the dispatch interval, \n
                          in MW, (as `np.float64`)
        ramp_up_rate      the ramp up rate, in MW/h, \n
                          (as `np.float64`)
        ramp_down_rate    the ramp down rate, in MW/h, \n
                          (as `np.float64`)
        ================  ========================================


    """
    last_interval_energy_dispatch = last_interval_dispatch[last_interval_dispatch['service'] == 'energy']
    last_interval_energy_dispatch = last_interval_energy_dispatch.loc[:, ['unit', 'dispatch']]
    last_interval_energy_dispatch.columns = ['unit', 'initial_output']
    ramp_rates = pd.merge(last_interval_energy_dispatch, ramp_rates, how='right', on='unit')
    ramp_rates = ramp_rates.fillna(0.0)
    return ramp_rates


def create_seed_ramp_rate_parameters(historical_dispatch, as_bid_ramp_rates):
    """Combine historical dispatch and as bid ramp rates to get seed ramp rate parameters for a time sequential model.

    Examples
    --------

    >>> historical_dispatch = pd.DataFrame({
    ... 'unit': ['A', 'B'],
    ... 'initial_output': [80.0, 100.0]})

    >>> as_bid_ramp_rates = pd.DataFrame({
    ... 'unit': ['A', 'B'],
    ... 'ramp_down_rate': [600.0, 1200.0],
    ... 'ramp_up_rate': [600.0, 1200.0]})

    >>> create_seed_ramp_rate_parameters(historical_dispatch,
    ...                                  as_bid_ramp_rates)
      unit  initial_output  ramp_down_rate  ramp_up_rate
    0    A            80.0           600.0         600.0
    1    B           100.0          1200.0        1200.0

    Parameters
    ----------
    historical_dispatch : pd.DataFrame

        ================  ========================================
        Columns:          Description:
        unit              unique identifier for units, (as `str`) \n
        initial_output    the output/consumption of the unit at \n
                          the start of the dispatch interval, \n
                          in MW, (as `np.float64`)
        ================  ========================================

    as_bid_ramp_rates

        ================  ========================================
        Columns:          Description:
        unit              unique identifier for units, (as `str`) \n
        ramp_up_rate      the ramp up rate, in MW/h, \n
                          (as `np.float64`)
        ramp_down_rate    the ramp down rate, in MW/h, \n
                          (as `np.float64`)
        ================  ========================================

    Returns
    -------
    pd.DataFrame

        ================  ========================================
        Columns:          Description:
        unit              unique identifier for units, (as `str`) \n
        initial_output    the output/consumption of the unit at \n
                          the start of the dispatch interval, \n
                          in MW, (as `np.float64`)
        ramp_up_rate      the ramp up rate, in MW/h, \n
                          (as `np.float64`)
        ramp_down_rate    the ramp down rate, in MW/h, \n
                          (as `np.float64`)
        ================  ========================================
    """
    return pd.merge(historical_dispatch, as_bid_ramp_rates, on='unit')

================
File: nempy\help_functions\helper_functions.py
================
import numpy as np
import pandas as pd


def save_index(dataframe, new_col_name, offset=0):
    # Make sure index starts at zero.
    dataframe = dataframe.reset_index(drop=True)
    # Save the indexes of the data frame as an np array.
    index_list = np.array(dataframe.index.values)
    # Add an offset to each element of the array.
    offset_index_list = index_list + offset
    # Add the list of indexes as a column to the data frame.
    dataframe[new_col_name] = offset_index_list
    return dataframe


def max_constraint_index(newest_variable_data):
    # Find the maximum constraint index already in use in the constraint matrix.
    max_index = newest_variable_data['ROWINDEX'].max()
    return max_index


def stack_columns(data_in, cols_to_keep, cols_to_stack, type_name, value_name):
    # Wrapping pd.melt to make it easier to use in nemlite context.
    stacked_data = pd.melt(data_in, id_vars=cols_to_keep, value_vars=cols_to_stack,
                           var_name=type_name, value_name=value_name)
    return stacked_data


def add_capacity_band_type(df_with_price_bands, ns):
    # Map the names of the capacity bands to a dataframe that already has the names of the price bands.
    band_map = pd.DataFrame()
    band_map[ns.col_price_band_number] = ns.cols_bid_price_name_list
    band_map[ns.col_capacity_band_number] = ns.cols_bid_cap_name_list
    df_with_capacity_and_price_bands = pd.merge(df_with_price_bands, band_map, 'left', [ns.col_price_band_number])
    return df_with_capacity_and_price_bands


def max_variable_index(newest_variable_data):
    # Find the maximum variable index already in use in the constraint matrix.
    max_index = newest_variable_data['INDEX'].max()
    return max_index


def update_rhs_values(constraint_rhs_and_type, new_rhs_values):
    if 'volume' in constraint_rhs_and_type.columns:
        rhs_name = 'volume'
    else:
        rhs_name = 'rhs'
    new_rhs_values = new_rhs_values.rename(columns={'rhs': 'new_rhs'})
    constraint_rhs_and_type = pd.merge(constraint_rhs_and_type, new_rhs_values, on='set', how='left')
    constraint_rhs_and_type[rhs_name] = np.where(~constraint_rhs_and_type['new_rhs'].isna(),
                                                 constraint_rhs_and_type['new_rhs'],
                                                 constraint_rhs_and_type[rhs_name])
    constraint_rhs_and_type = constraint_rhs_and_type.drop(columns=['new_rhs'])
    return constraint_rhs_and_type

================
File: nempy\spot_markert_backend\check.py
================
import numpy as np
import pandas as pd


def keep_details(fn):
    def wrapper(inner):
        inner.__name__ = fn.__name__
        inner.__doc__ = fn.__doc__
        return inner

    return wrapper


def energy_bid_ids_exist(func):
    @keep_details(func)
    def wrapper(*args):
        if 'bids' not in args[0]._decision_variables:
            raise ModelBuildError('This cannot be performed before energy volume bids are set.')
        func(*args)

    return wrapper


def all_units_have_info(func):
    @keep_details(func)
    def wrapper(*args):
        if not set(args[1]['unit'].unique()) <= set(args[0]._unit_info['unit']):
            raise ModelBuildError('Not all unit with bids are present in the unit_info input.')
        func(*args)
    return wrapper


def interconnectors_exist(func):
    @keep_details(func)
    def wrapper(*args):
        if 'interconnectors' not in args[0]._decision_variables:
            raise ModelBuildError('Losses cannot be added to interconnectors because they do not exist yet.')
        existing_inters = args[0]._decision_variables['interconnectors']['interconnector'].unique()
        new_inters = args[1]['interconnector'].unique()
        if not all(inter in existing_inters for inter in new_inters):
            raise ModelBuildError('Losses cannot be added to interconnectors because they do not exist yet.')
        func(*args)
    return wrapper


def bid_prices_monotonic_increasing(func, arg=1):
    @keep_details(func)
    def wrapper(*args):
        bids = args[arg].copy()
        if 'service' in bids.columns:
            bids = bids.set_index(['unit', 'service'], drop=True)
        else:
            bids = bids.set_index('unit', drop=True)
        bids = bids.transpose()
        bids.index = pd.to_numeric(bids.index)
        bids = bids.sort_index()
        for col in bids.columns:
            if not bids[col].is_monotonic:
                raise BidsNotMonotonicIncreasing('Bids of each unit are not monotonic increasing.')
        func(*args)

    return wrapper


def pre_dispatch(func):
    @keep_details(func)
    def wrapper(*args):
        if 'energy_bids' in args[0]._decision_variables and 'energy_bids' not in \
                args[0]._objective_function_components:
            raise ModelBuildError('No unit energy bids provided.')
        func(*args)

    return wrapper


def repeated_rows(name, cols, arg=1):
    def decorator(func):
        @keep_details(func)
        def wrapper(*args):
            cols_in_df = [col for col in cols if col in args[arg].columns]
            if args[0].check and len(args[arg].index) != len(args[arg].drop_duplicates(cols_in_df)):
                raise RepeatedRowError('{} should only have one row for each {}.'.format(name, ' '.join(cols_in_df)))
            func(*args)

        return wrapper

    return decorator


def column_data_types(name, dtypes, arg=1):
    def decorator(func):
        @keep_details(func)
        def wrapper(*args):
            if args[0].check:
                for column in args[arg].columns:
                    if column in dtypes and dtypes[column] == str:
                        if not all(args[arg].apply(lambda x: type(x[column]) == str, axis=1)):
                            raise ColumnDataTypeError('Column {} in {} should have type str'.format(column, name))
                    elif column in dtypes and dtypes[column] == 'callable':
                        if not all(args[arg].apply(lambda x: callable(x[column]), axis=1)):
                            raise ColumnDataTypeError('Column {} in {} should be a function'.format(column, name))
                    elif column in dtypes and dtypes[column] != args[arg][column].dtype:
                        raise ColumnDataTypeError('Column {} in {} should have type {}'.
                                                  format(column, name, dtypes[column]))
                    elif column not in dtypes and dtypes['else'] != args[arg][column].dtype:
                        raise ColumnDataTypeError('Column {} in {} should have type {}'.
                                                  format(column, name, dtypes['else']))
            func(*args)

        return wrapper

    return decorator


def required_columns(name, required, arg=1):
    def decorator(func):
        @keep_details(func)
        def wrapper(*args):
            if args[0].check:
                for column in required:
                    if column not in args[arg].columns:
                        raise MissingColumnError("Column '{}' not in {}.".format(column, name))
                if len(args[arg].columns) < 2:
                    raise MissingColumnError("No bid bands provided.")
            func(*args)

        return wrapper

    return decorator


def allowed_columns(name, allowed, arg=1):
    def decorator(func):
        @keep_details(func)
        def wrapper(*args):
            if args[0].check:
                for column in args[arg].columns:
                    if column not in allowed:
                        raise UnexpectedColumn("Column '{}' not allowed in {}.".format(column, name))
            func(*args)

        return wrapper

    return decorator


def column_values_must_be_real(name, cols_to_check, arg=1):
    def decorator(func):
        @keep_details(func)
        def wrapper(*args):
            if args[0].check:
                for column in cols_to_check:
                    if column not in args[arg].columns:
                        continue
                    if np.inf in args[arg][column].values:
                        raise ColumnValues("Value inf not allowed in column '{}' in {}.".format(column, name))
                    if np.NINF in args[arg][column].values:
                        raise ColumnValues("Value -inf not allowed in column '{}' in {}.".format(column, name))
                    if args[arg][column].isnull().any():
                        raise ColumnValues("Null values not allowed in column '{}' in {}.".format(column, name))
            func(*args)

        return wrapper

    return decorator


def column_values_not_negative(name, cols_to_check, arg=1):
    def decorator(func):
        @keep_details(func)
        def wrapper(*args):
            if args[0].check:
                for column in cols_to_check:
                    if column not in args[arg].columns:
                        continue
                    if args[arg][column].min() < 0.0:
                        raise ColumnValues("Negative values not allowed in column '{}' in {}.".format(column, name))
            func(*args)

        return wrapper

    return decorator


def column_values_outside_range(name, column_ranges, arg=1):
    def decorator(func):
        @keep_details(func)
        def wrapper(*args):
            if args[0].check:
                for column, allowed_range in column_ranges.items():
                    if not all(args[arg].apply(
                            lambda x: allowed_range[0] <= x[column] <= allowed_range[1], axis=1)):
                        raise ColumnValues(
                            "Values in {} in column '{}' outside the range {} to {}.".format(name, column,
                                                                                             allowed_range[0],
                                                                                             allowed_range[1]))
            func(*args)

        return wrapper

    return decorator


def table_exists(arg=1):
    def decorator(func):
        @keep_details(func)
        def wrapper(*args):
            with args[0].con:
                cur = args[0].con.cursor()
                check_query = ''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name='{}' '''
                cur.execute(check_query.format(args[1]))
                if cur.fetchone()[0] != 1:
                    raise MissingTable("The table {} does not exist.".format(args[1]))
            func(*args)

        return wrapper

    return decorator


class ModelBuildError(Exception):
    """Raise for building model components in wrong order."""


class RepeatedRowError(Exception):
    """Raise for repeated rows."""


class ColumnDataTypeError(Exception):
    """Raise for columns with incorrect data types."""


class MissingColumnError(Exception):
    """Raise for required column missing."""


class UnexpectedColumn(Exception):
    """Raise for unexpected column."""


class ColumnValues(Exception):
    """Raise for unexpected column."""


class BidsNotMonotonicIncreasing(Exception):
    """Raise for non monotonic increasing bids."""


class MissingTable(Exception):
    """Raise for trying to access missing table."""

================
File: nempy\spot_markert_backend\dataframe_validator.py
================
import numpy as np
import pandas as pd


class DataFrameSchema:
    def __init__(self, name, primary_keys=None, row_monatonic_increasing=None):
        self.name = name
        self.primary_keys = primary_keys
        self.columns = {}
        self.required_columns = []
        self.row_monatonic_increasing = row_monatonic_increasing

    def add_column(self, column, optional=False):
        self.columns[column.name] = column
        if not optional:
            self.required_columns.append(column.name)

    def validate(self, df):
        for col in df:
            if col not in self.columns:
                raise UnexpectedColumn("Column {} is not allowed in DataFrame {}.".format(col, self.name))

        for col in self.required_columns:
            if col not in df.columns:
                raise MissingColumnError("Column {} not in DataFrame {}.".format(col, self.name))

        for col in self.columns:
            if col in df.columns:
                self.columns[col].validate(df[col])

        if self.primary_keys is not None:
            self._check_for_repeated_rows(df)

    def _check_for_repeated_rows(self, df):
        cols_in_df = [col for col in self.primary_keys if col in df.columns]
        if len(df.index) != len(df.drop_duplicates(cols_in_df)):
            raise RepeatedRowError('{} should only have one row for each {}.'.format(self.name, ' '.join(cols_in_df)))

    def _check_row_monatonic_increasing(self, df):
        df = df.loc[:, self.row_monatonic_increasing]
        df = df.transpose()
        df.index = pd.to_numeric(df.index)
        df = df.sort_index()
        for col in df.columns:
            if not df[col].is_monotonic:
                raise BidsNotMonotonicIncreasing('Bids of each unit are not monotonic increasing.')


class SeriesSchema:
    def __init__(self, name, data_type, allowed_values=None, must_be_real_number=False, not_negative=False,
                 minimum=None, maximum=None):
        self.name = name
        self.data_type = data_type
        self.allowed_values = allowed_values
        self.must_be_real_number = must_be_real_number
        self.not_negative = not_negative
        self.min = minimum
        self.max = maximum

    def validate(self, series):
        self._check_data_type(series)
        self._check_allowed_values(series)
        self._check_is_real_number(series)
        self._check_is_not_negtaive(series)

    def _check_data_type(self, series):
        if self.data_type == str:
            if not all(series.apply(lambda x: type(x) == str)):
                raise ColumnDataTypeError('All elements of column {} should have type str'.format(self.name))
        elif self.data_type == callable:
            if not all(series.apply(lambda x: callable(x))):
                raise ColumnDataTypeError('All elements of column {} should have type callable'.format(self.name))
        elif self.data_type != series.dtype:
            raise ColumnDataTypeError('Column {} should have type {}'.format(self.name, self.data_type))

    def _check_allowed_values(self, series):
        if self.allowed_values is not None:
            if not series.isin(self.allowed_values).all():
                raise ColumnValues("The column {} can only contain the values {}.".format(self.name, self.allowed_values))

    def _check_is_real_number(self, series):
        if self.must_be_real_number:
            if np.inf in series.values:
                raise ColumnValues("Value inf not allowed in column {}.".format(self.name))
            if np.NINF in series.values:
                raise ColumnValues("Value -inf not allowed in column {}.".format(self.name))
            if series.isnull().any():
                raise ColumnValues("Null values not allowed in column {}.".format(self.name))

    def _check_is_not_negtaive(self, series):
        if self.not_negative:
            if series.min() < 0.0:
                raise ColumnValues("Negative values not allowed in column '{}'.".format(self.name))


class RepeatedRowError(Exception):
    """Raise for repeated rows."""


class ColumnDataTypeError(Exception):
    """Raise for columns with incorrect data types."""


class MissingColumnError(Exception):
    """Raise for required column missing."""


class UnexpectedColumn(Exception):
    """Raise for unexpected column."""


class ColumnValues(Exception):
    """Raise for unallowed column values."""


class BidsNotMonotonicIncreasing(Exception):
    """Raise for non monotonic increasing bids."""

================
File: nempy\spot_markert_backend\elastic_constraints.py
================
import pandas as pd
import numpy as np
from nempy.help_functions import helper_functions as hf


def create_deficit_variables(constraint_rhs, next_variable_id):
    """ Create variables that allow a constraint to violated at a specified cost.

    Examples
    --------

    >>> constraint_rhs = pd.DataFrame({
    ...   'constraint_id': [1, 2, 3],
    ...   'type': ['>=', '<=', '='],
    ...   'cost': [14000.0, 14000.0, 14000.]})

    >>> deficit_variables, lhs = create_deficit_variables(constraint_rhs, 1)

    Note two variables are needed for equality constraints, one to allow violation up and one to allow violation down.

    >>> print(deficit_variables)
       variable_id     cost  lower_bound  upper_bound        type
    0            1  14000.0          0.0          inf  continuous
    1            2  14000.0          0.0          inf  continuous
    0            3  14000.0          0.0          inf  continuous
    0            4  14000.0          0.0          inf  continuous

    >>> print(lhs)
       variable_id  constraint_id  coefficient
    0            1              1          1.0
    1            2              2         -1.0
    0            3              3         -1.0
    0            4              3          1.0

    Parameters
    ----------
    constraint_rhs : pd.DataFrame
        ==============  ====================================================================
        Columns:        Description:
        constraint_id   the id of the constraint (as `int`)
        type            the type of the constraint, e.g. ">=" or "<=" (as `str`)
        cost            the cost of using the deficit variable to violate the constraint (as `np.float64`)
        ==============  ====================================================================

    Returns
    -------
    deficit_variables : pd.DataFrame
        =============  ====================================================================
        Columns:       Description:
        variable_id    the id of the variable (as `int`)
        lower_bound    the minimum value of the variable (as `np.float64`)
        upper_bound    the maximum value of the variable (as `np.float64`)
        type           the type of variable, is continuous for deficit variables  (as `str`)
        cost           the cost of using the deficit variable to violate the constraint (as `np.float64`)
        =============  ====================================================================

    lhs : pd.DataFrame
        =============  ====================================================================
        Columns:       Description:
        variable_id    the id of the variable (as `int`)
        constraint_id  the id of the constraint (as `int`)
        coefficient    the variable lhs coefficient (as `np.float64`)
        =============  ====================================================================
    """

    inequalities = constraint_rhs[constraint_rhs['type'].isin(['>=', '<='])]
    equalities = constraint_rhs[constraint_rhs['type'] == '=']

    inequalities = hf.save_index(inequalities.reset_index(drop=True), 'variable_id', next_variable_id)

    inequalities_deficit_variables = inequalities.loc[:, ['variable_id', 'cost']]
    inequalities_deficit_variables['lower_bound'] = 0.0
    inequalities_deficit_variables['upper_bound'] = np.inf
    inequalities_deficit_variables['type'] = 'continuous'

    inequalities_lhs = inequalities.loc[:, ['variable_id', 'constraint_id', 'type']]
    inequalities_lhs['coefficient'] = np.where(inequalities_lhs['type'] == '>=', 1.0, -1.0)
    inequalities_lhs = inequalities_lhs.loc[:, ['variable_id', 'constraint_id', 'coefficient']]

    if not equalities.empty:
        if not inequalities.empty:
            next_variable_id = inequalities['variable_id'].max() + 1
        equalities_up = hf.save_index(equalities.reset_index(drop=True), 'variable_id', next_variable_id)
        next_variable_id = equalities_up['variable_id'].max() + 1
        equalities_down = hf.save_index(equalities.reset_index(drop=True), 'variable_id', next_variable_id)

        equalities_up_deficit_variables = equalities_up.loc[:, ['variable_id', 'cost']]
        equalities_up_deficit_variables['lower_bound'] = 0.0
        equalities_up_deficit_variables['upper_bound'] = np.inf
        equalities_up_deficit_variables['type'] = 'continuous'

        equalities_down_deficit_variables = equalities_down.loc[:, ['variable_id', 'cost']]
        equalities_down_deficit_variables['lower_bound'] = 0.0
        equalities_down_deficit_variables['upper_bound'] = np.inf
        equalities_down_deficit_variables['type'] = 'continuous'

        equalities_up_lhs = equalities_up.loc[:, ['variable_id', 'constraint_id', 'type']]
        equalities_up_lhs['coefficient'] = -1.0
        equalities_up_lhs = equalities_up_lhs.loc[:, ['variable_id', 'constraint_id', 'coefficient']]

        equalities_down_lhs = equalities_down.loc[:, ['variable_id', 'constraint_id', 'type']]
        equalities_down_lhs['coefficient'] = 1.0
        equalities_down_lhs = equalities_down_lhs.loc[:, ['variable_id', 'constraint_id', 'coefficient']]

        deficit_variables = pd.concat([inequalities_deficit_variables, equalities_up_deficit_variables,
                                       equalities_down_deficit_variables])

        lhs = pd.concat([inequalities_lhs, equalities_up_lhs, equalities_down_lhs])

    else:
        deficit_variables = inequalities_deficit_variables
        lhs = inequalities_lhs

    return deficit_variables, lhs

================
File: nempy\spot_markert_backend\fcas_constraints.py
================
import pandas as pd
import numpy as np
from nempy.help_functions import helper_functions as hf


def joint_ramping_constraints_raise_reg(unit_limits, unit_info, dispatch_interval, next_constraint_id):
    constraint_settings = {'generator': {'type': '<=',
                                         'reg_lhs_coefficient': 1.0,
                                         'ramp_direction': 1.0,
                                         'reg_service': 'raise_reg'},
                           'load': {'type': '>=',
                                    'reg_lhs_coefficient': -1.0,
                                    'ramp_direction': -1.0,
                                    'reg_service': 'raise_reg'}
                           }
    rhs_and_type, variable_mapping = \
        joint_ramping_constraints_load_and_generator_constructor(unit_limits, unit_info, dispatch_interval,
                                                                 next_constraint_id, constraint_settings)
    return rhs_and_type, variable_mapping


def joint_ramping_constraints_lower_reg(unit_limits, unit_info, dispatch_interval, next_constraint_id):
    constraint_settings = {'generator': {'type': '>=',
                                         'reg_lhs_coefficient': -1.0,
                                         'ramp_direction': -1.0,
                                         'reg_service': 'lower_reg'},
                           'load':  {'type': '<=',
                                     'reg_lhs_coefficient': 1.0,
                                     'ramp_direction': 1.0,
                                     'reg_service': 'lower_reg'}
                           }
    rhs_and_type, variable_mapping = \
        joint_ramping_constraints_load_and_generator_constructor(unit_limits, unit_info, dispatch_interval,
                                                                 next_constraint_id, constraint_settings)
    return rhs_and_type, variable_mapping


def joint_ramping_constraints_load_and_generator_constructor(unit_limits, unit_info, dispatch_interval,
                                                             next_constraint_id, settings):
    constraints = hf.save_index(unit_limits, 'constraint_id', next_constraint_id)
    constraints = pd.merge(constraints, unit_info, 'left', on='unit')
    constraints_generators = constraints[constraints['dispatch_type'] == 'generator'].copy()
    constraints_loads = constraints[constraints['dispatch_type'] == 'load'].copy()
    gen_rhs_and_type, gen_variable_mapping = \
        joint_ramping_constraints_generic_constructor(constraints_generators, settings['generator'],
                                                      dispatch_interval)
    load_rhs_and_type, load_variable_mapping = \
        joint_ramping_constraints_generic_constructor(constraints_loads, settings['load'],
                                                      dispatch_interval)
    rhs_and_type = pd.concat([gen_rhs_and_type, load_rhs_and_type])
    variable_mapping = pd.concat([gen_variable_mapping, load_variable_mapping])
    return rhs_and_type, variable_mapping


def joint_ramping_constraints_generic_constructor(constraints, settings, dispatch_interval):
    constraints['rhs'] = constraints['initial_output'] + settings['ramp_direction'] * \
                         (constraints['ramp_rate'] * dispatch_interval / 60)
    constraints['service'] = settings['reg_service']
    constraints['type'] = settings['type']
    rhs_and_type = constraints.loc[:, ['unit', 'constraint_id', 'type', 'rhs']]
    variable_mapping_reg = constraints.loc[:, ['constraint_id', 'unit', 'service']]
    variable_mapping_reg['coefficient'] = settings['reg_lhs_coefficient']
    variable_mapping_energy = constraints.loc[:, ['constraint_id', 'unit', 'service']]
    variable_mapping_energy['service'] = 'energy'
    variable_mapping_energy['coefficient'] = 1.0
    variable_mapping = pd.concat([variable_mapping_reg, variable_mapping_energy])
    return rhs_and_type, variable_mapping


def joint_capacity_constraints(contingency_trapeziums, unit_info, next_constraint_id):
    """Creates constraints to ensure there is adequate capacity for contingency, regulation and energy dispatch targets.

    Create two constraints for each contingency services, one ensures operation on upper slope of the fcas contingency
    trapezium is consistent with regulation raise and energy dispatch, the second ensures operation on lower slope of
    the fcas contingency trapezium is consistent with regulation lower and energy dispatch.

    The constraints are described in the
    :download:`FCAS MODEL IN NEMDE documentation section 6.2  <../../docs/pdfs/FCAS Model in NEMDE.pdf>`.

    Examples
    --------
    >>> import pandas as pd

    >>> contingency_trapeziums = pd.DataFrame({
    ... 'unit': ['A'],
    ... 'service': ['raise_6s'],
    ... 'max_availability': [60.0],
    ... 'enablement_min': [20.0],
    ... 'low_break_point': [40.0],
    ... 'high_break_point': [60.0],
    ... 'enablement_max': [80.0]})

    >>> unit_info = pd.DataFrame({
    ... 'unit': ['A'],
    ... 'dispatch_type': ['generator']})

    >>> next_constraint_id = 1

    >>> type_and_rhs, variable_mapping = joint_capacity_constraints(contingency_trapeziums, unit_info,
    ...                                                             next_constraint_id)

    >>> print(type_and_rhs)
      unit   service  constraint_id type   rhs
    0    A  raise_6s              1   <=  80.0
    0    A  raise_6s              2   >=  20.0

    >>> print(variable_mapping)
       constraint_id unit    service  coefficient
    0              1    A     energy     1.000000
    0              1    A   raise_6s     0.333333
    0              1    A  raise_reg     1.000000
    0              2    A     energy     1.000000
    0              2    A   raise_6s    -0.333333
    0              2    A  lower_reg    -1.000000

    Parameters
    ----------
    contingency_trapeziums : pd.DataFrame
        The FCAS trapeziums for the contingency services being offered.

        ================   ======================================================================
        Columns:           Description:
        unit               unique identifier of a dispatch unit (as `str`)
        service            the contingency service being offered (as `str`)
        max_availability   the maximum volume of the contingency service in MW (as `np.float64`)
        enablement_min     the energy dispatch level at which the unit can begin to provide the
                           contingency service, in MW (as `np.float64`)
        low_break_point    the energy dispatch level at which the unit can provide the full
                           contingency service offered, in MW (as `np.float64`)
        high_break_point   the energy dispatch level at which the unit can no longer provide the
                           full contingency service offered, in MW (as `np.float64`)
        enablement_max     the energy dispatch level at which the unit can no longer begin
                           the contingency service, in MW (as `np.float64`)
        ================   ======================================================================

    unit_info : pd.DataFrame

        ================   ======================================================================
        Columns:           Description:
        unit               unique identifier of a dispatch unit (as `str`)
        dispatch_type      "load" or "generator" (as `str`)
        ================   ======================================================================


    next_constraint_id : int
        The next integer to start using for constraint ids

    Returns
    -------
    type_and_rhs : pd.DataFrame
        The type and rhs of each constraint.

        =============  ====================================================================
        Columns:       Description:
        unit           unique identifier of a dispatch unit (as `str`)
        service        the regulation service the constraint is associated with (as `str`)
        constraint_id  the id of the variable (as `int`)
        type           the type of the constraint, e.g. "=" (as `str`)
        rhs            the rhs of the constraint (as `np.float64`)
        =============  ====================================================================

    variable_map : pd.DataFrame
        The type of variables that should appear on the lhs of the constraint.

        =============  ==========================================================================
        Columns:       Description:
        constraint_id  the id of the constraint (as `np.int64`)
        unit           the unit variables the constraint should map too (as `str`)
        service        the service type of the variables the constraint should map to (as `str`)
        the constraint factor in the lhs coefficient (as `np.float64`)
        =============  ==========================================================================

    """

    # Create each constraint set.
    contingency_trapeziums = pd.merge(contingency_trapeziums, unit_info, 'inner', on='unit')
    constraints_upper_slope = hf.save_index(contingency_trapeziums, 'constraint_id', next_constraint_id)
    next_constraint_id = max(constraints_upper_slope['constraint_id']) + 1
    constraints_lower_slope = hf.save_index(contingency_trapeziums, 'constraint_id', next_constraint_id)

    # Calculate the slope coefficients for the constraints.
    constraints_upper_slope['upper_slope_coefficient'] = ((constraints_upper_slope['enablement_max'] -
                                                           constraints_upper_slope['high_break_point']) /
                                                          constraints_upper_slope['max_availability'])
    constraints_lower_slope['lower_slope_coefficient'] = ((constraints_lower_slope['low_break_point'] -
                                                           constraints_lower_slope['enablement_min']) /
                                                          constraints_lower_slope['max_availability'])

    # Define the direction of the upper slope constraints and the rhs value.
    constraints_upper_slope['type'] = '<='
    constraints_upper_slope['rhs'] = constraints_upper_slope['enablement_max']
    type_and_rhs_upper_slope = constraints_upper_slope.loc[:, ['unit', 'service', 'constraint_id', 'type', 'rhs']]

    # Define the direction of the lower slope constraints and the rhs value.
    constraints_lower_slope['type'] = '>='
    # constraints_lower_slope['enablement_min'] = np.where(constraints_lower_slope['enablement_min'] == 0.0, 1.0,
    #                                                      constraints_lower_slope['enablement_min'])
    constraints_lower_slope['rhs'] = constraints_lower_slope['enablement_min']
    type_and_rhs_lower_slope = constraints_lower_slope.loc[:, ['unit', 'service', 'constraint_id', 'type', 'rhs']]

    # Define the variables on the lhs of the upper slope constraints and their coefficients.
    energy_mapping_upper_slope = constraints_upper_slope.loc[:, ['constraint_id', 'unit']]
    energy_mapping_upper_slope['service'] = 'energy'
    energy_mapping_upper_slope['coefficient'] = 1.0
    contingency_mapping_upper_slope = constraints_upper_slope.loc[:, ['constraint_id', 'unit', 'service',
                                                                      'upper_slope_coefficient']]
    contingency_mapping_upper_slope = \
        contingency_mapping_upper_slope.rename(columns={"upper_slope_coefficient": "coefficient"})
    regulation_mapping_upper_slope = constraints_upper_slope.loc[:, ['constraint_id', 'unit', 'dispatch_type']]
    regulation_mapping_upper_slope['service'] = np.where(regulation_mapping_upper_slope['dispatch_type'] == 'generator',
                                                         'raise_reg', 'lower_reg')
    regulation_mapping_upper_slope = regulation_mapping_upper_slope.drop('dispatch_type', axis=1)
    regulation_mapping_upper_slope['coefficient'] = 1.0

    # Define the variables on the lhs of the lower slope constraints and their coefficients.
    energy_mapping_lower_slope = constraints_lower_slope.loc[:, ['constraint_id', 'unit']]
    energy_mapping_lower_slope['service'] = 'energy'
    energy_mapping_lower_slope['coefficient'] = 1.0
    contingency_mapping_lower_slope = constraints_lower_slope.loc[:, ['constraint_id', 'unit', 'service',
                                                                      'lower_slope_coefficient']]
    contingency_mapping_lower_slope = \
        contingency_mapping_lower_slope.rename(columns={"lower_slope_coefficient": "coefficient"})
    contingency_mapping_lower_slope['coefficient'] = -1 * contingency_mapping_lower_slope['coefficient']
    regulation_mapping_lower_slope = constraints_lower_slope.loc[:, ['constraint_id', 'unit', 'dispatch_type']]
    regulation_mapping_lower_slope['service'] = np.where(regulation_mapping_lower_slope['dispatch_type'] == 'generator',
                                                         'lower_reg', 'raise_reg')
    regulation_mapping_lower_slope = regulation_mapping_lower_slope.drop('dispatch_type', axis=1)
    regulation_mapping_lower_slope['coefficient'] = -1.0

    # Combine type_and_rhs and variable_mapping.
    type_and_rhs = pd.concat([type_and_rhs_upper_slope, type_and_rhs_lower_slope])
    variable_mapping = pd.concat([energy_mapping_upper_slope, contingency_mapping_upper_slope,
                                  regulation_mapping_upper_slope, energy_mapping_lower_slope,
                                  contingency_mapping_lower_slope, regulation_mapping_lower_slope])
    return type_and_rhs, variable_mapping


def energy_and_regulation_capacity_constraints(regulation_trapeziums, next_constraint_id):
    """Creates constraints to ensure there is adequate capacity for regulation and energy dispatch targets.

    Create two constraints for each regulation services, one ensures operation on upper slope of the fcas contingency
    trapezium is consistent with energy dispatch, the second ensures operation on lower slope of the fcas regulation
    trapezium is consistent with energy dispatch.

    The constraints are described in the
    :download:`FCAS MODEL IN NEMDE documentation section 6.3  <../../docs/pdfs/FCAS Model in NEMDE.pdf>`.

    Examples
    --------
    >>> import pandas as pd

    >>> regulation_trapeziums = pd.DataFrame({
    ... 'unit': ['A'],
    ... 'service': ['raise_reg'],
    ... 'max_availability': [60.0],
    ... 'enablement_min': [20.0],
    ... 'low_break_point': [40.0],
    ... 'high_break_point': [60.0],
    ... 'enablement_max': [80.0]})

    >>> next_constraint_id = 1

    >>> type_and_rhs, variable_mapping = energy_and_regulation_capacity_constraints(regulation_trapeziums,
    ...                                                                             next_constraint_id)

    >>> print(type_and_rhs)
      unit    service  constraint_id type   rhs
    0    A  raise_reg              1   <=  80.0
    0    A  raise_reg              2   >=  20.0

    >>> print(variable_mapping)
       constraint_id unit    service  coefficient
    0              1    A     energy     1.000000
    0              1    A  raise_reg     0.333333
    0              2    A     energy     1.000000
    0              2    A  raise_reg    -0.333333

    Parameters
    ----------
    regulation_trapeziums : pd.DataFrame
        The FCAS trapeziums for the regulation services being offered.

    ================   ======================================================================
    Columns:           Description:
    unit               unique identifier of a dispatch unit (as `str`)
    service            the regulation service being offered (as `str`)
    max_availability   the maximum volume of the contingency service in MW (as `np.float64`)
    enablement_min     the energy dispatch level at which the unit can begin to provide the
                       contingency service, in MW (as `np.float64`)
    low_break_point    the energy dispatch level at which the unit can provide the full
                       contingency service offered, in MW (as `np.float64`)
    high_break_point   the energy dispatch level at which the unit can no longer provide the
                       full contingency service offered, in MW (as `np.float64`)
    enablement_max     the energy dispatch level at which the unit can no longer begin
                       the contingency service, in MW (as `np.float64`)
    ================   ======================================================================


    next_constraint_id : int
        The next integer to start using for constraint ids

    Returns
    -------
    type_and_rhs : pd.DataFrame
        The type and rhs of each constraint.

        =============  ====================================================================
        Columns:       Description:
        unit           unique identifier of a dispatch unit (as `str`)
        service        the regulation service the constraint is associated with (as `str`)
        constraint_id  the id of the variable (as `int`)
        type           the type of the constraint, e.g. "=" (as `str`)
        rhs            the rhs of the constraint (as `np.float64`)
        =============  ====================================================================

    variable_map : pd.DataFrame
        The type of variables that should appear on the lhs of the constraint.

        =============  ==========================================================================
        Columns:       Description:
        constraint_id  the id of the constraint (as `np.int64`)
        unit           the unit variables the constraint should map too (as `str`)
        service        the service type of the variables the constraint should map to (as `str`)
        coefficient    the constraint factor in the lhs coefficient (as `np.float64`)
        =============  ==========================================================================

    """

    # Create each constraint set.
    constraints_upper_slope = hf.save_index(regulation_trapeziums, 'constraint_id', next_constraint_id)
    next_constraint_id = max(constraints_upper_slope['constraint_id']) + 1
    constraints_lower_slope = hf.save_index(regulation_trapeziums, 'constraint_id', next_constraint_id)

    # Calculate the slope coefficients for the constraints.
    constraints_upper_slope['upper_slope_coefficient'] = ((constraints_upper_slope['enablement_max'] -
                                                           constraints_upper_slope['high_break_point']) /
                                                          constraints_upper_slope['max_availability'])
    constraints_lower_slope['lower_slope_coefficient'] = ((constraints_lower_slope['low_break_point'] -
                                                           constraints_lower_slope['enablement_min']) /
                                                          constraints_lower_slope['max_availability'])

    # Define the direction of the upper slope constraints and the rhs value.
    constraints_upper_slope['type'] = '<='
    constraints_upper_slope['rhs'] = constraints_upper_slope['enablement_max']
    type_and_rhs_upper_slope = constraints_upper_slope.loc[:, ['unit', 'service', 'constraint_id', 'type', 'rhs']]

    # Define the direction of the lower slope constraints and the rhs value.
    constraints_lower_slope['type'] = '>='
    # constraints_lower_slope['enablement_min'] = np.where(constraints_lower_slope['enablement_min'] == 0.0, 1.0,
    #                                                      constraints_lower_slope['enablement_min'])
    constraints_lower_slope['rhs'] = constraints_lower_slope['enablement_min']
    type_and_rhs_lower_slope = constraints_lower_slope.loc[:, ['unit', 'service', 'constraint_id', 'type', 'rhs']]

    # Define the variables on the lhs of the upper slope constraints and their coefficients.
    energy_mapping_upper_slope = constraints_upper_slope.loc[:, ['constraint_id', 'unit']]
    energy_mapping_upper_slope['service'] = 'energy'
    energy_mapping_upper_slope['coefficient'] = 1.0
    regulation_mapping_upper_slope = constraints_upper_slope.loc[:, ['constraint_id', 'unit', 'service',
                                                                     'upper_slope_coefficient']]
    regulation_mapping_upper_slope = \
        regulation_mapping_upper_slope.rename(columns={"upper_slope_coefficient": "coefficient"})

    # Define the variables on the lhs of the lower slope constraints and their coefficients.
    energy_mapping_lower_slope = constraints_lower_slope.loc[:, ['constraint_id', 'unit']]
    energy_mapping_lower_slope['service'] = 'energy'
    energy_mapping_lower_slope['coefficient'] = 1.0
    regulation_mapping_lower_slope = constraints_lower_slope.loc[:, ['constraint_id', 'unit', 'service',
                                                                     'lower_slope_coefficient']]
    regulation_mapping_lower_slope = \
        regulation_mapping_lower_slope.rename(columns={"lower_slope_coefficient": "coefficient"})
    regulation_mapping_lower_slope['coefficient'] = -1 * regulation_mapping_lower_slope['coefficient']

    # Combine type_and_rhs and variable_mapping.
    type_and_rhs = pd.concat([type_and_rhs_upper_slope, type_and_rhs_lower_slope])
    variable_mapping = pd.concat([energy_mapping_upper_slope, regulation_mapping_upper_slope,
                                  energy_mapping_lower_slope, regulation_mapping_lower_slope])
    return type_and_rhs, variable_mapping

================
File: nempy\spot_markert_backend\interconnectors.py
================
import numpy as np
import pandas as pd
from nempy.help_functions import helper_functions as hf


def create(definitions, next_variable_id):
    """Create decision variables, and their mapping to constraints. For modeling interconnector flows. As DataFrames.

    Examples
    --------
    Definitions for two interconnectors, one called A, that nominal flows from region X to region Y, note A can flow in
    both directions because of the way max and min are defined. The interconnector B nominal flows from Y to Z, but can
    only flow in the forward direction.

    >>> pd.options.display.width = None

    >>> inter_definitions = pd.DataFrame({
    ...   'interconnector': ['A', 'B'],
    ...   'link': ['A', 'B'],
    ...   'from_region': ['X', 'Y'],
    ...   'to_region': ['Y', 'Z'],
    ...   'max': [100.0, 400.0],
    ...   'min': [-100.0, 50.0],
    ...   'generic_constraint_factor': [1, 1],
    ...   'from_region_loss_factor': [0.9, 1.0],
    ...   'to_region_loss_factor': [1.0, 1.1]})

    >>> print(inter_definitions)
      interconnector link from_region to_region    max    min  generic_constraint_factor  from_region_loss_factor  to_region_loss_factor
    0              A    A           X         Y  100.0 -100.0                          1                      0.9                    1.0
    1              B    B           Y         Z  400.0   50.0                          1                      1.0                    1.1

    Start creating new variable ids from 0.

    >>> next_variable_id = 0

    Run the function and print results.

    >>> decision_variables, constraint_map = create(inter_definitions, next_variable_id)

    >>> print(decision_variables)
      interconnector link  variable_id  lower_bound  upper_bound        type  generic_constraint_factor
    0              A    A            0       -100.0        100.0  continuous                          1
    1              B    B            1         50.0        400.0  continuous                          1

    >>> print(constraint_map)
       variable_id interconnector link region service  coefficient
    0            0              A    A      Y  energy          1.0
    1            1              B    B      Z  energy          1.1
    2            0              A    A      X  energy         -0.9
    3            1              B    B      Y  energy         -1.0

    """

    # Create a variable_id for each interconnector.
    decision_variables = hf.save_index(definitions, 'variable_id', next_variable_id)

    # Create two entries in the constraint_map for each interconnector. This means the variable will be mapped to the
    # demand constraint of both connected regions.
    constraint_map = hf.stack_columns(decision_variables, ['variable_id', 'interconnector', 'link', 'max', 'min'],
                                      ['to_region', 'from_region'], 'direction', 'region')
    loss_factors = hf.stack_columns(decision_variables, ['variable_id'],
                                    ['from_region_loss_factor', 'to_region_loss_factor'], 'direction', 'loss_factor')
    loss_factors['direction'] = loss_factors['direction'].apply(lambda x: x.replace('_loss_factor', ''))
    constraint_map = pd.merge(constraint_map, loss_factors, on=['variable_id', 'direction'])

    # Define decision variable attributes.
    decision_variables['type'] = 'continuous'
    decision_variables = decision_variables.loc[:, ['interconnector', 'link', 'variable_id', 'min', 'max', 'type',
                                                    'generic_constraint_factor']]
    decision_variables.columns = ['interconnector', 'link',  'variable_id', 'lower_bound', 'upper_bound', 'type',
                                  'generic_constraint_factor']

    # Set positive coefficient for the to_region so the interconnector flowing in the nominal direction helps meet the
    # to_region demand constraint. Negative for the from_region, same logic.
    constraint_map['coefficient'] = np.where(constraint_map['direction'] == 'to_region',
                                             1.0 * constraint_map['loss_factor'],
                                             -1.0 * constraint_map['loss_factor'])
    constraint_map['service'] = 'energy'
    constraint_map = constraint_map.loc[:, ['variable_id', 'interconnector', 'link',
                                            'region', 'service', 'coefficient']]

    return decision_variables, constraint_map


def link_inter_loss_to_interpolation_weights(weight_variables, loss_variables, loss_functions, next_constraint_id):
    """
    Examples
    --------

    Setup function inputs

    >>> loss_variables = pd.DataFrame({
    ...   'interconnector': ['I'],
    ...   'link': ['I'],
    ...   'variable_id': [0]})

    >>> weight_variables = pd.DataFrame({
    ...   'interconnector': ['I', 'I', 'I'],
    ...   'link': ['I', 'I', 'I'],
    ...   'variable_id': [1, 2, 3],
    ...   'break_point': [-100.0, 0, 100.0]})

    Loss functions can arbitrary, they just need to take the flow as input and return losses as an output.

    >>> def constant_losses(flow):
    ...     return abs(flow) * 0.05

    The loss function get assigned to an interconnector by its row in the loss functions DataFrame.

    >>> loss_functions = pd.DataFrame({
    ...    'interconnector': ['I'],
    ...    'link': ['I'],
    ...    'from_region_loss_share': [0.5],
    ...    'loss_function': [constant_losses]})

    >>> next_constraint_id = 0

    Create the constraints.

    >>> lhs, rhs = link_inter_loss_to_interpolation_weights(weight_variables, loss_variables, loss_functions,
    ...                                                     next_constraint_id)

    >>> print(lhs)
       variable_id  constraint_id  coefficient
    0            1              0          5.0
    1            2              0          0.0
    2            3              0          5.0

    >>> print(rhs)
      interconnector link  constraint_id type  rhs_variable_id
    0              I    I              0    =                0
    """

    # Create a constraint for each set of weight variables.
    constraint_ids = weight_variables.loc[:, ['interconnector', 'link']].drop_duplicates(['interconnector', 'link'])
    constraint_ids = hf.save_index(constraint_ids, 'constraint_id', next_constraint_id)

    # Map weight variables to their corresponding constraints.
    lhs = pd.merge(weight_variables.loc[:, ['interconnector', 'link', 'variable_id', 'break_point']],
                   constraint_ids, 'inner', on=['interconnector', 'link'])
    lhs = pd.merge(lhs, loss_functions.loc[:, ['interconnector', 'link', 'loss_function']], 'inner',
                   on=['interconnector', 'link'])

    # Evaluate the loss function at each break point to get the lhs coefficient.
    lhs['coefficient'] = lhs.apply(lambda x: x['loss_function'](x['break_point']), axis=1)
    lhs = lhs.loc[:, ['variable_id', 'constraint_id', 'coefficient']]

    # Get the loss variables that will be on the rhs of the constraints.
    rhs_variables = loss_variables.loc[:, ['interconnector', 'link', 'variable_id']]
    rhs_variables.columns = ['interconnector', 'link', 'rhs_variable_id']
    # Map the rhs variables to their constraints.
    rhs = pd.merge(constraint_ids, rhs_variables, 'inner', on=['interconnector', 'link'])
    rhs['type'] = '='
    rhs = rhs.loc[:, ['interconnector', 'link', 'constraint_id', 'type', 'rhs_variable_id']]
    return lhs, rhs


def link_weights_to_inter_flow(weight_variables, flow_variables, next_constraint_id):
    """
    Examples
    --------

    Setup function inputs

    >>> flow_variables = pd.DataFrame({
    ...   'interconnector': ['I'],
    ...   'link': ['I'],
    ...   'variable_id': [0]})

    >>> weight_variables = pd.DataFrame({
    ...   'interconnector': ['I', 'I', 'I'],
    ...   'link': ['I', 'I', 'I'],
    ...   'variable_id': [1, 2, 3],
    ...   'break_point': [-100.0, 0, 100.0]})

    >>> next_constraint_id = 0

    Create the constraints.

    >>> lhs, rhs = link_weights_to_inter_flow(weight_variables, flow_variables, next_constraint_id)

    >>> print(lhs)
       variable_id  constraint_id  coefficient
    0            1              0       -100.0
    1            2              0          0.0
    2            3              0        100.0

    >>> print(rhs)
      interconnector link  constraint_id type  rhs_variable_id
    0              I    I              0    =                0
    """

    # Create a constraint for each set of weight variables.
    constraint_ids = weight_variables.loc[:, ['interconnector', 'link']].drop_duplicates(['interconnector', 'link'])
    constraint_ids = hf.save_index(constraint_ids, 'constraint_id', next_constraint_id)

    # Map weight variables to their corresponding constraints.
    lhs = pd.merge(weight_variables.loc[:, ['interconnector', 'link', 'variable_id', 'break_point']],
                   constraint_ids, 'inner', on=['interconnector', 'link'])
    lhs['coefficient'] = lhs['break_point']
    lhs = lhs.loc[:, ['variable_id', 'constraint_id', 'coefficient']]

    # Get the interconnector variables that will be on the rhs of constraint.
    rhs_variables = flow_variables.loc[:, ['interconnector', 'link', 'variable_id']]
    rhs_variables.columns = ['interconnector', 'link', 'rhs_variable_id']
    # Map the rhs variables to their constraints.
    rhs = pd.merge(constraint_ids, rhs_variables, 'inner', on=['interconnector', 'link'])
    rhs['type'] = '='
    rhs = rhs.loc[:, ['interconnector', 'link', 'constraint_id', 'type', 'rhs_variable_id']]
    return lhs, rhs


def create_weights_must_sum_to_one(weight_variables, next_constraint_id):
    """Create the constraint to force weight variable to sum to one, need for interpolation to work.

    For one interconnector, if we had  three weight variables w1, w2, and w3, then the constraint would be of the form.

        w1 * 1.0 + w2 * 1.0 + w3 * 1.0 = 1.0

    Examples
    --------

    Setup function inputs

    >>> weight_variables = pd.DataFrame({
    ...   'interconnector': ['I', 'I', 'I'],
    ...   'link': ['I', 'I', 'I'],
    ...   'variable_id': [1, 2, 3],
    ...   'break_point': [-100.0, 0, 100.0]})

    >>> next_constraint_id = 0

    Create the constraints.

    >>> lhs, rhs = create_weights_must_sum_to_one(weight_variables, next_constraint_id)

    >>> print(lhs)
       variable_id  constraint_id  coefficient
    0            1              0          1.0
    1            2              0          1.0
    2            3              0          1.0

    >>> print(rhs)
      interconnector link  constraint_id type  rhs
    0              I    I              0    =  1.0

    """

    # Create a constraint for each set of weight variables.
    constraint_ids = weight_variables.loc[:, ['interconnector', 'link']].drop_duplicates(['interconnector', 'link'])
    constraint_ids = hf.save_index(constraint_ids, 'constraint_id', next_constraint_id)

    # Map weight variables to their corresponding constraints.
    lhs = pd.merge(weight_variables.loc[:, ['interconnector', 'link', 'variable_id']], constraint_ids,
                   'inner', on=['interconnector', 'link'])
    lhs['coefficient'] = 1.0
    lhs = lhs.loc[:, ['variable_id', 'constraint_id', 'coefficient']]

    # Create rhs details for each constraint.
    rhs = constraint_ids
    rhs['type'] = '='
    rhs['rhs'] = 1.0
    return lhs, rhs


def create_weights(break_points, next_variable_id):
    """Create interpolation weight variables for each breakpoint.

    Examples
    --------

    >>> break_points = pd.DataFrame({
    ...   'interconnector': ['I', 'I', 'I'],
    ...   'loss_segment': [1, 2, 3],
    ...   'break_point': [-100.0, 0.0, 100.0]})

    >>> next_variable_id = 0

    >>> weight_variables = create_weights(break_points, next_variable_id)

    >>> print(weight_variables.loc[:, ['interconnector', 'loss_segment', 'break_point', 'variable_id']])
      interconnector  loss_segment  break_point  variable_id
    0              I             1       -100.0            0
    1              I             2          0.0            1
    2              I             3        100.0            2

    >>> print(weight_variables.loc[:, ['variable_id', 'lower_bound', 'upper_bound', 'type']])
       variable_id  lower_bound  upper_bound        type
    0            0          0.0          1.0  continuous
    1            1          0.0          1.0  continuous
    2            2          0.0          1.0  continuous

    Parameters
    ----------
    break_points : pd.DataFrame
        ==============  ================================================================================
        Columns:        Description:
        interconnector  unique identifier of a interconnector (as `str`)
        loss_segment    unique identifier of a loss segment on an interconnector basis (as `np.float64`)
        break_points    the interconnector flow values to interpolate losses between (as `np.float64`)
        ==============  ================================================================================

    next_variable_id : int

    Returns
    -------
    weight_variables : pd.DataFrame

        ==============  ==============================================================================
        Columns:        Description:
        interconnector  unique identifier of a interconnector (as `str`)
        loss_segment    unique identifier of a loss segment on an interconnector basis (as `np.float64`)
        break_points    the interconnector flow values to interpolate losses between (as `np.int64`)
        variable_id     the id of the variable (as `np.int64`)
        lower_bound    the lower bound of the variable, is zero for weight variables (as `np.float64`)
        upper_bound    the upper bound of the variable, is one for weight variables (as `np.float64`)
        type           the type of variable, is continuous for bids  (as `str`)
        ==============  ==============================================================================
    """
    # Create a variable for each break point.
    weight_variables = hf.save_index(break_points, 'variable_id', next_variable_id)
    weight_variables['lower_bound'] = 0.0
    weight_variables['upper_bound'] = 1.0
    weight_variables['type'] = 'continuous'
    return weight_variables


def create_loss_variables(inter_variables, inter_constraint_map, loss_shares, next_variable_id):
    """
    Examples
    --------
    Setup function inputs

    >>> inter_variables = pd.DataFrame({
    ...   'interconnector': ['I'],
    ...   'link': ['i'],
    ...   'lower_bound': [-50.0],
    ...   'upper_bound': [100.0],
    ...   'type': ['continuous']})

    >>> inter_constraint_map = pd.DataFrame({
    ... 'interconnector': ['I', 'I'],
    ... 'link': ['i', 'i'],
    ... 'region': ['X', 'Y'],
    ... 'service': ['energy', 'energy'],
    ... 'coefficient': [1.0, -1.0]})

    >>> loss_shares = pd.DataFrame({
    ...    'interconnector': ['I'],
    ...    'link': ['i'],
    ...    'from_region_loss_share': [0.5],
    ...    'from_region': ['X']})

    >>> next_constraint_id = 0

    Create the constraints.

    >>> loss_variables, constraint_map = create_loss_variables(inter_variables, inter_constraint_map, loss_shares,
    ...                                                        next_constraint_id)

    >>> print(loss_variables)
      interconnector link  variable_id  lower_bound  upper_bound        type
    0              I    i            0       -100.0        100.0  continuous

    >>> print(constraint_map)
       variable_id region service  coefficient
    0            0      X  energy         -0.5
    1            0      Y  energy         -0.5
    """

    # Preserve the interconnector variable id for merging later.
    columns_for_loss_variables = inter_variables.loc[:, ['interconnector', 'link', 'lower_bound',
                                                         'upper_bound', 'type']]
    columns_for_loss_variables['upper_bound'] = \
        columns_for_loss_variables.loc[:, ['lower_bound', 'upper_bound']].abs().max(axis=1)
    columns_for_loss_variables['lower_bound'] = -1 * columns_for_loss_variables['upper_bound']

    inter_constraint_map = inter_constraint_map.loc[:, ['interconnector', 'link', 'region', 'service', 'coefficient']]

    # Create a variable id for loss variables
    loss_variables = hf.save_index(columns_for_loss_variables, 'variable_id', next_variable_id)
    loss_variables = pd.merge(loss_variables,
                              loss_shares.loc[:, ['interconnector', 'link', 'from_region_loss_share', 'from_region']],
                              on=['interconnector', 'link'])

    # Create the loss variable constraint map by combining the new variables and the flow variable constraint map.
    constraint_map = pd.merge(
        loss_variables.loc[:, ['variable_id', 'interconnector', 'link', 'from_region_loss_share', 'from_region']],
        inter_constraint_map, 'inner', on=['interconnector', 'link'])

    # Assign losses to regions according to the from_region_loss_share
    constraint_map['coefficient'] = np.where(constraint_map['from_region'] == constraint_map['region'],
                                             - 1 * constraint_map['from_region_loss_share'],
                                             - 1 * (1 - constraint_map['from_region_loss_share']))

    loss_variables = loss_variables.loc[:, ['interconnector', 'link', 'variable_id',
                                            'lower_bound', 'upper_bound', 'type']]
    constraint_map = constraint_map.loc[:, ['variable_id', 'region', 'service', 'coefficient']]
    return loss_variables, constraint_map

================
File: nempy\spot_markert_backend\market_constraints.py
================
from nempy.help_functions import helper_functions as hf
import pandas as pd


def energy(demand, next_constraint_id):
    """Create the constraints that ensure the amount of supply dispatched in each region equals demand.

    If only one region exists then the constraint will be of the form:

        unit 1 output + unit 2 output +. . .+ unit n output = region demand

    If multiple regions exist then a constraint will ne created for each region. If there were 2 units A and B in region
    X, and 2 units C and D in region Y, then the constraints would be of the form:

        constraint 1: unit A output + unit B output = region X demand
        constraint 2: unit C output + unit D output = region Y demand

    Examples
    --------

    >>> import pandas

    Defined the unit capacities.

    >>> demand = pd.DataFrame({
    ...   'region': ['X', 'Y'],
    ...   'demand': [1000.0, 2000.0]})

    >>> next_constraint_id = 0

    Create the constraint information.

    >>> type_and_rhs, variable_map = energy(demand, next_constraint_id)

    >>> print(type_and_rhs)
      region  constraint_id type     rhs
    0      X              0    =  1000.0
    1      Y              1    =  2000.0

    >>> print(variable_map)
       constraint_id region service  coefficient
    0              0      X  energy          1.0
    1              1      Y  energy          1.0

    Parameters
    ----------
    demand : pd.DataFrame
        Demand by region.

        ========  =====================================================================================
        Columns:  Description:
        region    unique identifier of a region (as `str`)
        demand    the non dispatchable demand, in MW (as `np.float64`)
        ========  =====================================================================================

    next_constraint_id : int
        The next integer to start using for constraint ids.


    Returns
    -------
    type_and_rhs : pd.DataFrame
        The type and rhs of each constraint.

        =============  ===============================================================
        Columns:       Description:
        region         unique identifier of a market region (as `str`)
        constraint_id  the id of the variable (as `int`)
        type           the type of the constraint, e.g. "=" (as `str`)
        rhs            the rhs of the constraint (as `np.float64`)
        =============  ===============================================================

    variable_map : pd.DataFrame
        The type of variables that should appear on the lhs of the constraint.

        =============  ==========================================================================
        Columns:       Description:
        constraint_id  the id of the constraint (as `np.int64`)
        region         the regional variables the constraint should map too (as `str`)
        service        the service type of the variables the constraint should map to (as `str`)
        coefficient    the upper bound of the variable, the volume bid (as `np.float64`)
        =============  ==========================================================================
    """
    # Create an index for each constraint.
    type_and_rhs = hf.save_index(demand, 'constraint_id', next_constraint_id)
    type_and_rhs['type'] = '='  # Supply and interconnector flow must exactly equal demand.
    type_and_rhs['rhs'] = type_and_rhs['demand']
    type_and_rhs = type_and_rhs.loc[:, ['region', 'constraint_id', 'type', 'rhs']]

    # Map constraints to energy variables in their region.
    variable_map = type_and_rhs.loc[:, ['constraint_id', 'region']]
    variable_map['service'] = 'energy'
    variable_map['coefficient'] = 1.0
    return type_and_rhs, variable_map


def fcas(fcas_requirements, next_constraint_id):
    """Create the constraints that ensure the amount of FCAS supply dispatched  equals requirements.

    Examples
    --------

    >>> import pandas

    Defined the unit capacities.

    >>> fcas_requirements = pd.DataFrame({
    ...     'set': ['raise_reg_main', 'raise_reg_main', 'raise_reg_main', 'raise_reg_main'],
    ...     'service': ['raise_reg', 'raise_reg', 'raise_reg', 'raise_reg'],
    ...     'region': ['QLD', 'NSW', 'VIC', 'SA'],
    ...     'volume': [100.0, 100.0, 100.0, 100.0]})

    >>> next_constraint_id = 0

    Create the constraint information.

    >>> type_and_rhs, variable_map = fcas(fcas_requirements, next_constraint_id)

    >>> print(type_and_rhs)
                  set  constraint_id type    rhs
    0  raise_reg_main              0    =  100.0

    >>> print(variable_map)
       constraint_id    service region  coefficient
    0              0  raise_reg    QLD          1.0
    1              0  raise_reg    NSW          1.0
    2              0  raise_reg    VIC          1.0
    3              0  raise_reg     SA          1.0

    Parameters
    ----------
    fcas_requirements : pd.DataFrame
        requirement by set and the regions and service the requirement applies to.

        ========  ===================================================================
        Columns:  Description:
        set       unique identifier of the requirement set (as `str`)
        service   the service or services the requirement set applies to (as `str`)
        region    unique identifier of a region (as `str`)
        volume    the amount of service required, in MW (as `np.float64`)
        type      the direction of the constrain '=', '>=' or '<=', optional, a \n
                  value of '=' is assumed if the column is missing (as `str`)
        ========  ===================================================================

    next_constraint_id : int
        The next integer to start using for constraint ids.

    Returns
    -------
    type_and_rhs : pd.DataFrame
        The type and rhs of each constraint.

        =============  ===================================================================
        Columns:       Description:
        set            unique identifier of a market region (as `str`)
        constraint_id  the id of the variable (as `int`)
        type           the type of the constraint, e.g. "=" (as `str`)
        rhs            the rhs of the constraint (as `np.float64`)
        =============  ===================================================================

    variable_map : pd.DataFrame
        The type of variables that should appear on the lhs of the constraint.

        =============  ==========================================================================
        Columns:       Description:
        constraint_id  the id of the constraint (as `np.int64`)
        region         the regional variables the constraint should map too (as `str`)
        service        the service type of the variables the constraint should map to (as `str`)
        coefficient    the upper bound of the variable, the volume bid (as `np.float64`)
        =============  ==========================================================================
    """
    # Set default value if optional column is missing.
    if 'type' not in fcas_requirements.columns:
        fcas_requirements['type'] = '='

    # Create an index for each constraint.
    type_and_rhs = fcas_requirements.loc[:, ['set', 'volume', 'type']]
    type_and_rhs = type_and_rhs.drop_duplicates('set')
    type_and_rhs = hf.save_index(type_and_rhs, 'constraint_id', next_constraint_id)
    type_and_rhs['rhs'] = type_and_rhs['volume']
    type_and_rhs = type_and_rhs.loc[:, ['set', 'constraint_id', 'type', 'rhs']]

    # Map constraints to energy variables in their region.
    variable_map = fcas_requirements.loc[:, ['set', 'service', 'region']]
    variable_map = pd.merge(variable_map, type_and_rhs.loc[:, ['set', 'constraint_id']], 'inner', on='set')
    variable_map['coefficient'] = 1.0
    variable_map = variable_map.loc[:, ['constraint_id', 'service', 'region', 'coefficient']]
    return type_and_rhs, variable_map

================
File: nempy\spot_markert_backend\objective_function.py
================
import pandas as pd
import numpy as np
from nempy.help_functions import helper_functions as hf


def bids(variable_ids, price_bids, unit_info):
    """Create the cost coefficients of energy in bids in the objective function.

    This function defines the cost associated with each decision variable that represents a unit's energy bid. Costs are
    are with reference to the regional node.
    """
    # If no service column is provided assume bids are for energy.
    if 'service' not in price_bids.columns:
        price_bids['service'] = 'energy'

    # Get the list of columns that are bid bands.
    bid_bands = [col for col in price_bids.columns if col not in ['unit', 'service']]
    price_bids = hf.stack_columns(price_bids, cols_to_keep=['unit', 'service'], cols_to_stack=bid_bands,
                                  type_name='capacity_band', value_name='cost')
    # Match bid cost with existing variable ids
    objective_function = pd.merge(variable_ids, price_bids, how='inner', on=['unit', 'service', 'capacity_band'])
    objective_function = pd.merge(objective_function, unit_info.loc[:, ['unit', 'dispatch_type']], how='inner',
                                  on=['unit'])
    objective_function['cost'] = np.where((objective_function['dispatch_type'] == 'load') &
                                          (objective_function['service'] == 'energy'),
                                          -1.0 * objective_function['cost'], objective_function['cost'])
    return objective_function


def scale_by_loss_factors(objective_function, unit_info):
    """
    Scale the bid cost by dividing by the loss factor.

    Parameters
    ----------
    objective_function : pd.DataFrame
        Cost by variable id, also including unit and capacity band so loss factors can be applied if provided.

        =============  ===============================================================
        Columns:       Description:
        unit           unique identifier of a dispatch unit (as `str`)
        capacity_band  the bid band of the variable (as `str`)
        variable_id    the id of the variable (as `int`)
        =============  ===============================================================

    unit_info : pd.DataFrame
        The loss factor to scale bids by.

        =============  ===============================================================
        Columns:       Description:
        unit           unique identifier of a dispatch unit (as `str`)
        loss_factor    the id of the variable (as `int`)
        =============  ===============================================================

    Returns
    -------
    pd.DataFrame

        =============  ===============================================================
        Columns:       Description:
        unit           unique identifier of a dispatch unit (as `str`)
        capacity_band  the bid band of the variable (as `str`)
        variable_id    the id of the variable (as `int`)
        cost           the bid cost of the variable (as `float`)
        =============  ===============================================================
    """

    # Match units with their loss factors.
    objective_function = pd.merge(objective_function, unit_info, how='inner', on='unit')
    # Refer bids cost to regional reference node, if a loss factor  was provided.
    objective_function['cost'] = np.where(objective_function['service'] == 'energy',
                                          objective_function['cost'] / objective_function['loss_factor'],
                                          objective_function['cost'])
    return objective_function

================
File: nempy\spot_markert_backend\solver_interface.py
================
import numpy as np
import pandas as pd
from mip import Model, xsum, minimize, CONTINUOUS, OptimizationStatus, BINARY, CBC, GUROBI, LP_Method


class InterfaceToSolver:
    """A wrapper for the mip model class, allows interaction with mip using pd.DataFrames."""

    def __init__(self, solver_name='CBC'):
        self.variables = {}
        self.linear_mip_variables = {}

        self.solver_name = solver_name
        if solver_name == 'CBC':
            self.mip_model = Model("market", solver_name=CBC)
            self.linear_mip_model = Model("market", solver_name=CBC)
        elif solver_name == 'GUROBI':
            self.mip_model = Model("market", solver_name=GUROBI)
            self.linear_mip_model = Model("market", solver_name=GUROBI)
        else:
            raise ValueError("Solver '{}' not recognised.")

        self.mip_model.verbose = 0
        self.mip_model.solver.set_mip_gap_abs(1e-10)
        self.mip_model.solver.set_mip_gap(1e-20)
        self.mip_model.lp_method = LP_Method.DUAL

        self.linear_mip_model.verbose = 0
        self.linear_mip_model.solver.set_mip_gap_abs(1e-10)
        self.linear_mip_model.solver.set_mip_gap(1e-20)
        self.linear_mip_model.lp_method = LP_Method.DUAL

    def add_variables(self, decision_variables):
        """Add decision variables to the model.

        Examples
        --------
        >>> decision_variables = pd.DataFrame({
        ...   'variable_id': [0, 1],
        ...   'lower_bound': [0.0, 0.0],
        ...   'upper_bound': [6.0, 1.0],
        ...   'type': ['continuous', 'binary']})

        >>> si = InterfaceToSolver()

        >>> si.add_variables(decision_variables)

        The underlying mip_model should now have 2 variables.

        >>> print(si.mip_model.num_cols)
        2

        The first one should have the following properties.

        >>> print(si.mip_model.var_by_name('0').var_type)
        C

        >>> print(si.mip_model.var_by_name('0').lb)
        0.0

        >>> print(si.mip_model.var_by_name('0').ub)
        6.0

        The second one should have the following properties.

        >>> print(si.mip_model.var_by_name('1').var_type)
        B

        >>> print(si.mip_model.var_by_name('1').lb)
        0.0

        >>> print(si.mip_model.var_by_name('1').ub)
        1.0

        """
        # Create a mapping between the nempy level names for variable types and the mip representation.
        variable_types = {'continuous': CONTINUOUS, 'binary': BINARY}
        # Add each variable to the mip model.
        for variable_id, lower_bound, upper_bound, variable_type in zip(
                list(decision_variables['variable_id']), list(decision_variables['lower_bound']),
                list(decision_variables['upper_bound']), list(decision_variables['type'])):
            self.variables[variable_id] = self.mip_model.add_var(lb=lower_bound, ub=upper_bound,
                                                                 var_type=variable_types[variable_type],
                                                                 name=str(variable_id))

            self.linear_mip_variables[variable_id] = self.linear_mip_model.add_var(lb=lower_bound, ub=upper_bound,
                                                                                   var_type=variable_types[
                                                                                       variable_type],
                                                                                   name=str(variable_id))

    def add_sos_type_2(self, sos_variables, sos_id_columns, position_column):
        """Add groups of special ordered sets of type 2 two the mip model.

        Examples
        --------

        >>> decision_variables = pd.DataFrame({
        ...   'variable_id': [0, 1, 2, 3, 4, 5],
        ...   'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
        ...   'upper_bound': [5.0, 5.0, 5.0, 5.0, 5.0, 5.0],
        ...   'type': ['continuous', 'continuous', 'continuous',
        ...            'continuous', 'continuous', 'continuous']})

        >>> sos_variables = pd.DataFrame({
        ...   'variable_id': [0, 1, 2, 3, 4, 5],
        ...   'sos_id': ['A', 'A', 'A', 'B', 'B', 'B'],
        ...   'position': [0, 1, 2, 0, 1, 2]})

        >>> si = InterfaceToSolver()

        >>> si.add_variables(decision_variables)

        >>> si.add_sos_type_2(sos_variables, 'sos_id', 'position')

        """

        # Function that adds sets to mip model.
        def add_sos_vars(sos_group):
            self.mip_model.add_sos(list(zip(sos_group['vars'], sos_group[position_column])), 2)

        # For each variable_id get the variable object from the mip model
        sos_variables['vars'] = sos_variables['variable_id'].apply(lambda x: self.variables[x])
        # Break up the sets based on their id and add them to the model separately.
        sos_variables.groupby(sos_id_columns).apply(add_sos_vars)
        # This is a hack to make sure mip knows there are binary constraints.
        self.mip_model.add_var(var_type=BINARY, obj=0.0)

    def add_sos_type_1(self, sos_variables):
        # Function that adds sets to mip model.
        def add_sos_vars(sos_group):
            self.mip_model.add_sos(list(zip(sos_group['vars'], [1.0 for i in range(len(sos_variables['vars']))])), 1)

        # For each variable_id get the variable object from the mip model
        sos_variables['vars'] = sos_variables['variable_id'].apply(lambda x: self.variables[x])
        # Break up the sets based on their id and add them to the model separately.
        sos_variables.groupby('sos_id').apply(add_sos_vars)
        # This is a hack to make mip knows there are binary constraints.
        self.mip_model.add_var(var_type=BINARY, obj=0.0)

    def add_objective_function(self, objective_function):
        """Add the objective function to the mip model.

        Examples
        --------

        >>> decision_variables = pd.DataFrame({
        ...   'variable_id': [0, 1, 2, 3, 4, 5],
        ...   'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
        ...   'upper_bound': [5.0, 5.0, 5.0, 5.0, 5.0, 5.0],
        ...   'type': ['continuous', 'continuous', 'continuous',
        ...            'continuous', 'continuous', 'continuous']})

        >>> objective_function = pd.DataFrame({
        ...   'variable_id': [0, 1, 3, 4, 5],
        ...   'cost': [1.0, 2.0, -1.0, 5.0, 0.0]})

        >>> si = InterfaceToSolver()

        >>> si.add_variables(decision_variables)

        >>> si.add_objective_function(objective_function)

        >>> print(si.mip_model.var_by_name('0').obj)
        1.0

        >>> print(si.mip_model.var_by_name('5').obj)
        0.0

        """
        objective_function = objective_function.sort_values('variable_id')
        objective_function = objective_function.set_index('variable_id')
        obj = minimize(xsum(objective_function['cost'][i] * self.variables[i] for i in
                            list(objective_function.index)))
        self.mip_model.objective = obj
        self.linear_mip_model.objective = obj

    def add_constraints(self, constraints_lhs, constraints_type_and_rhs):
        """Add constraints to the mip model.

        Examples
        --------
        >>> decision_variables = pd.DataFrame({
        ...   'variable_id': [0, 1, 2, 3, 4, 5],
        ...   'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
        ...   'upper_bound': [5.0, 5.0, 10.0, 10.0, 5.0, 5.0],
        ...   'type': ['continuous', 'continuous', 'continuous',
        ...            'continuous', 'continuous', 'continuous']})

        >>> constraints_lhs = pd.DataFrame({
        ...   'constraint_id': [1, 1, 2, 2],
        ...   'variable_id': [0, 1, 3, 4],
        ...   'coefficient': [1.0, 0.5, 1.0, 2.0]})

        >>> constraints_type_and_rhs = pd.DataFrame({
        ...   'constraint_id': [1, 2],
        ...   'type': ['<=', '='],
        ...   'rhs': [10.0, 20.0]})

        >>> si = InterfaceToSolver()

        >>> si.add_variables(decision_variables)

        >>> si.add_constraints(constraints_lhs, constraints_type_and_rhs)

        >>> print(si.mip_model.constr_by_name('1'))
        1: +1.0 0 +0.5 1 <= 10.0

        >>> print(si.mip_model.constr_by_name('2'))
        2: +1.0 3 +2.0 4 = 20.0

        """

        constraints_lhs = constraints_lhs.groupby(['constraint_id', 'variable_id'], as_index=False).agg(
            {'coefficient': 'sum'})
        rows = constraints_lhs.groupby(['constraint_id'], as_index=False)

        # Make a dictionary so constraint rhs values can be accessed using the constraint id.
        rhs = dict(zip(constraints_type_and_rhs['constraint_id'], constraints_type_and_rhs['rhs']))
        # Make a dictionary so constraint type can be accessed using the constraint id.
        enq_type = dict(zip(constraints_type_and_rhs['constraint_id'], constraints_type_and_rhs['type']))
        var_ids = constraints_lhs['variable_id'].to_numpy()
        vars = np.asarray(
            [self.variables[k] if k in self.variables.keys() else None for k in range(0, max(var_ids) + 1)])
        coefficients = constraints_lhs['coefficient'].to_numpy()
        for row_id, row in rows.indices.items():
            # Use the variable_ids to get mip variable objects present in the constraints
            lhs_variables = vars[var_ids[row]]
            # Use the positions of the non nan values to the lhs coefficients.
            lhs = coefficients[row]
            # Multiply and the variables by their coefficients and sum to create the lhs of the constraint.
            exp = lhs_variables * lhs
            exp = exp.tolist()
            exp = xsum(exp)
            # Add based on inequality type.
            if enq_type[row_id] == '<=':
                new_constraint = exp <= rhs[row_id]
            elif enq_type[row_id] == '>=':
                new_constraint = exp >= rhs[row_id]
            elif enq_type[row_id] == '=':
                new_constraint = exp == rhs[row_id]
            else:
                raise ValueError("Constraint type not recognised should be one of '<=', '>=' or '='.")
            self.mip_model.add_constr(new_constraint, name=str(row_id))
            self.linear_mip_model.add_constr(new_constraint, name=str(row_id))

    def optimize(self):
        """Optimize the mip model.

        If an optimal solution cannot be found and the investigate_infeasibility flag is set to True then remove
        constraints until a feasible solution is found.

        Examples
        --------
        >>> decision_variables = pd.DataFrame({
        ...   'variable_id': [0, 1, 2, 3, 4, 5],
        ...   'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
        ...   'upper_bound': [5.0, 5.0, 10.0, 10.0, 5.0, 5.0],
        ...   'type': ['continuous', 'continuous', 'continuous',
        ...            'continuous', 'continuous', 'continuous']})

        >>> constraints_lhs = pd.DataFrame({
        ...   'constraint_id': [1, 1, 2, 2],
        ...   'variable_id': [0, 1, 3, 4],
        ...   'coefficient': [1.0, 0.5, 1.0, 2.0]})

        >>> constraints_type_and_rhs = pd.DataFrame({
        ...   'constraint_id': [1, 2],
        ...   'type': ['<=', '='],
        ...   'rhs': [10.0, 20.0]})

        >>> si = InterfaceToSolver()

        >>> si.add_variables(decision_variables)

        >>> si.add_constraints(constraints_lhs, constraints_type_and_rhs)

        >>> si.optimize()

        >>> decision_variables['value'] = si.get_optimal_values_of_decision_variables(decision_variables)

        >>> print(decision_variables)
           variable_id  lower_bound  upper_bound        type  value
        0            0          0.0          5.0  continuous    0.0
        1            1          0.0          5.0  continuous    0.0
        2            2          0.0         10.0  continuous    0.0
        3            3          0.0         10.0  continuous   10.0
        4            4          0.0          5.0  continuous    5.0
        5            5          0.0          5.0  continuous    0.0
        """
        status = self.mip_model.optimize()
        if status != OptimizationStatus.OPTIMAL:
            # Attempt find constraint causing infeasibility.
            print('Model infeasible attempting to find problem constraint.')
            con_index = find_problem_constraint(self.mip_model)
            print('Couldn\'t find an optimal solution, but removing con {} fixed INFEASIBLITY'.format(con_index))
            raise ValueError('Linear program infeasible')

    def get_optimal_values_of_decision_variables(self, variable_definitions):
        """Get the optimal values for each decision variable.

        Examples
        --------

        >>> decision_variables = pd.DataFrame({
        ...   'variable_id': [0, 1, 2, 3, 4, 5],
        ...   'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
        ...   'upper_bound': [5.0, 5.0, 10.0, 10.0, 5.0, 5.0],
        ...   'type': ['continuous', 'continuous', 'continuous',
        ...            'continuous', 'continuous', 'continuous']})

        >>> constraints_lhs = pd.DataFrame({
        ...   'constraint_id': [1, 1, 2, 2],
        ...   'variable_id': [0, 1, 3, 4],
        ...   'coefficient': [1.0, 0.5, 1.0, 2.0]})

        >>> constraints_type_and_rhs = pd.DataFrame({
        ...   'constraint_id': [1, 2],
        ...   'type': ['<=', '='],
        ...   'rhs': [10.0, 20.0]})

        >>> si = InterfaceToSolver()

        >>> si.add_variables(decision_variables)

        >>> si.add_constraints(constraints_lhs, constraints_type_and_rhs)

        >>> si.optimize()

        >>> decision_variables['value'] = si.get_optimal_values_of_decision_variables(decision_variables)

        >>> print(decision_variables)
           variable_id  lower_bound  upper_bound        type  value
        0            0          0.0          5.0  continuous    0.0
        1            1          0.0          5.0  continuous    0.0
        2            2          0.0         10.0  continuous    0.0
        3            3          0.0         10.0  continuous   10.0
        4            4          0.0          5.0  continuous    5.0
        5            5          0.0          5.0  continuous    0.0

        """
        values = variable_definitions['variable_id'].apply(lambda x: self.mip_model.var_by_name(str(x)).x)
        return values

    def get_optimal_values_of_decision_variables_lin(self, variable_definitions):
        values = variable_definitions['variable_id'].apply(lambda x: self.linear_mip_model.var_by_name(str(x)).x)
        return values

    def get_slack_in_constraints(self, constraints_type_and_rhs):
        """Get the slack values in each constraint.

        Examples
        --------

        >>> decision_variables = pd.DataFrame({
        ...   'variable_id': [0, 1, 2, 3, 4, 5],
        ...   'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
        ...   'upper_bound': [5.0, 5.0, 10.0, 10.0, 5.0, 5.0],
        ...   'type': ['continuous', 'continuous', 'continuous',
        ...            'continuous', 'continuous', 'continuous']})

        >>> constraints_lhs = pd.DataFrame({
        ...   'constraint_id': [1, 1, 2, 2],
        ...   'variable_id': [0, 1, 3, 4],
        ...   'coefficient': [1.0, 0.5, 1.0, 2.0]})

        >>> constraints_type_and_rhs = pd.DataFrame({
        ...   'constraint_id': [1, 2],
        ...   'type': ['<=', '='],
        ...   'rhs': [10.0, 20.0]})

        >>> si = InterfaceToSolver()

        >>> si.add_variables(decision_variables)

        >>> si.add_constraints(constraints_lhs, constraints_type_and_rhs)

        >>> si.optimize()

        >>> constraints_type_and_rhs['slack'] = si.get_slack_in_constraints(constraints_type_and_rhs)

        >>> print(constraints_type_and_rhs)
           constraint_id type   rhs  slack
        0              1   <=  10.0   10.0
        1              2    =  20.0    0.0

        """
        slack = constraints_type_and_rhs['constraint_id'].apply(
            lambda x: getattr(self.mip_model.constr_by_name(str(x)), "slack", 0.0))
        return slack

    def price_constraints(self, constraint_ids_to_price):
        """For each constraint_id find the marginal value of the constraint.

        This is done by incrementing the constraint by a value of 1.0 and re-optimizing the model, the marginal cost
        of the constraint is increase in the objective function value between model runs.

        Examples
        --------

        >>> decision_variables = pd.DataFrame({
        ...   'variable_id': [0, 1, 2, 3, 4, 5],
        ...   'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
        ...   'upper_bound': [5.0, 5.0, 10.0, 10.0, 5.0, 5.0],
        ...   'type': ['continuous', 'continuous', 'continuous',
        ...            'continuous', 'continuous', 'continuous']})

        >>> objective_function = pd.DataFrame({
        ...   'variable_id': [0, 1, 2, 3, 4, 5],
        ...   'cost': [1.0, 3.0, 10.0, 8.0, 9.0, 7.0]})

        >>> constraints_lhs = pd.DataFrame({
        ...   'constraint_id': [1, 1, 1, 1],
        ...   'variable_id': [0, 1, 3, 4],
        ...   'coefficient': [1.0, 1.0, 1.0, 1.0]})

        >>> constraints_type_and_rhs = pd.DataFrame({
        ...   'constraint_id': [1],
        ...   'type': ['='],
        ...   'rhs': [20.0]})

        >>> si = InterfaceToSolver()

        >>> si.add_variables(decision_variables)

        >>> si.add_constraints(constraints_lhs, constraints_type_and_rhs)

        >>> si.add_objective_function(objective_function)

        >>> si.optimize()

        >>> si.linear_mip_model.optimize()
        <OptimizationStatus.OPTIMAL: 0>

        >>> prices = si.price_constraints([1])

        >>> print(prices)
        {1: 8.0}

        >>> decision_variables['value'] = si.get_optimal_values_of_decision_variables(decision_variables)

        >>> print(decision_variables)
           variable_id  lower_bound  upper_bound        type  value
        0            0          0.0          5.0  continuous    5.0
        1            1          0.0          5.0  continuous    5.0
        2            2          0.0         10.0  continuous    0.0
        3            3          0.0         10.0  continuous   10.0
        4            4          0.0          5.0  continuous    0.0
        5            5          0.0          5.0  continuous    0.0

        """
        costs = {}
        for id in constraint_ids_to_price:
            costs[id] = self.linear_mip_model.constr_by_name(str(id)).pi
        return costs

    def update_rhs(self, constraint_id, violation_degree):
        constraint = self.linear_mip_model.constr_by_name(str(constraint_id))
        constraint.rhs += violation_degree

    def update_variable_bounds(self, new_bounds):
        for variable_id, lb, ub in zip(new_bounds['variable_id'], new_bounds['lower_bound'], new_bounds['upper_bound']):
            self.mip_model.var_by_name(str(variable_id)).lb = lb
            self.mip_model.var_by_name(str(variable_id)).ub = ub

    def disable_variables(self, variables):
        for var_id in variables['variable_id']:
            var = self.linear_mip_model.var_by_name(str(var_id))
            var.lb = 0.0
            var.ub = 0.0


def find_problem_constraint(base_prob):
    cons = []
    test_prob = base_prob.copy()
    for con in [con.name for con in base_prob.constrs]:
        [test_prob.remove(c) for c in test_prob.constrs if c.name == con]
        status = test_prob.optimize()
        cons.append(con)
        if status == OptimizationStatus.OPTIMAL:
            return cons
    return []


def create_lhs(constraints, decision_variables, join_columns):
    """Combine constraints with general definitions of lhs with variables to give an explicit lhs definition.

    Both constraints and decision_variables can have a coefficient, the coefficient use in the actual lhs will
    be the product of the two coefficients.

    Examples
    --------

    >>> decision_variables = pd.DataFrame({
    ...   'variable_id': [0, 1, 2, 3, 4, 5],
    ...   'region': ['NSW', 'NSW', 'VIC',
    ...              'VIC', 'VIC', 'VIC'],
    ...   'service': ['energy', 'energy','energy',
    ...               'energy','energy','energy',],
    ...   'coefficient': [0.9, 0.8, 1.0, 0.95, 1.1, 1.01]})

    >>> constraints = pd.DataFrame({
    ...   'constraint_id': [1, 2],
    ...   'region': ['NSW', 'VIC'],
    ...   'service': ['energy', 'energy'],
    ...   'coefficient': [1.0, 1.0]})

    >>> lhs = create_lhs(decision_variables, constraints, ['region', 'service'])

    >>> print(lhs)
       constraint_id  variable_id  coefficient
    0              1            0         0.90
    1              1            1         0.80
    2              2            2         1.00
    3              2            3         0.95
    4              2            4         1.10
    5              2            5         1.01

    Parameters
    ----------
    constraints : pd.DataFrame

        =============  ===============================================================
        Columns:       Description:
        constraint_id  the unique identifier of the constraint (as `np.int64`)
        join_columns   one or more columns defining the types of variables that should
                       be on the lhs (as `str`)
        coefficient    the constraint level contribution to the lhs coefficient (as `np.float64`)
        =============  ===============================================================

    decision_variables : pd.DataFrame

        =============  ===============================================================
        Columns:       Description:
        variable_id    the unique identifier of the variable (as `np.int64`)
        join_columns   one or more columns defining the types of variables that should
                       be on the lhs (as `str`)
        coefficient    the variable level contribution to the lhs coefficient (as `np.float64`)
        =============  ===============================================================

    Returns
    -------
    lhs : pd.DataFrame

        =============  ===============================================================
        Columns:       Description:
        constraint_id  the unique identifier of the constraint (as `np.int64`)
        variable_id    the unique identifier of the variable (as `np.int64`)
        coefficient    the constraint level contribution to the lhs coefficient (as `np.float64`)
        =============  ===============================================================
    """
    constraints = pd.merge(constraints, decision_variables, 'inner', on=join_columns)
    constraints['coefficient'] = constraints['coefficient_x'] * constraints['coefficient_y']
    lhs = constraints.loc[:, ['constraint_id', 'variable_id', 'coefficient']]
    return lhs


def create_mapping_of_generic_constraint_sets_to_constraint_ids(constraints, market_constraints):
    """Combine generic constraints and fcas market constraints to get the full set of generic constraints.

    Returns non if there are no generic of fcas market constraints.

    Examples
    --------

    >>> constraints = {
    ...   'generic': pd.DataFrame({
    ...       'constraint_id': [0, 1],
    ...       'set': ['A', 'B']})
    ...   }

    >>> market_constraints = {
    ...   'fcas': pd.DataFrame({
    ...       'constraint_id': [2, 3],
    ...       'set': ['C', 'D']})
    ...   }

    >>> generic_constraints = create_mapping_of_generic_constraint_sets_to_constraint_ids(
    ... constraints, market_constraints)

    >>> print(generic_constraints)
       constraint_id set
    0              0   A
    1              1   B
    0              2   C
    1              3   D


    Parameters
    ----------
    constraints : dict{str : pd.DataFrame}

        The pd.DataFrame stored under the key 'generic', if it exists, should have the structure.

        =============  ===============================================================
        Columns:       Description:
        constraint_id  the unique identifier of the constraint (as `np.int64`)
        set            the constraint set that the id refers to (as `str`)
        =============  ===============================================================

    market_constraints : dict{str : pd.DataFrame}

        The pd.DataFrame stored under the key 'fcas', if it exists, should have the structure.

        =============  ===============================================================
        Columns:       Description:
        constraint_id  the unique identifier of the constraint (as `np.int64`)
        set            the constraint set that the id refers to (as `str`)
        =============  ===============================================================

    Returns
    -------
    pd.DataFrame or None

        If pd.DataFrame

        =============  ===============================================================
        Columns:       Description:
        constraint_id  the unique identifier of the constraint (as `np.int64`)
        set            the constraint set that the id refers to (as `str`)
        =============  ===============================================================

    """
    generic_constraints = []
    if 'generic' in constraints:
        generic_constraints.append(constraints['generic'].loc[:, ['constraint_id', 'set']])
    if 'fcas' in market_constraints:
        generic_constraints.append(market_constraints['fcas'].loc[:, ['constraint_id', 'set']])
    if len(generic_constraints) > 0:
        return pd.concat(generic_constraints)
    else:
        return None


def create_unit_level_generic_constraint_lhs(generic_constraint_units, generic_constraint_ids,
                                             unit_bids_to_constraint_map):
    """Find the lhs variables from units for generic constraints.

    Examples
    --------

    >>> generic_constraint_units = pd.DataFrame({
    ...   'set': ['A', 'A'],
    ...   'unit': ['X', 'Y'],
    ...   'service': ['energy', 'energy'],
    ...   'coefficient': [0.9, 0.8]})

    >>> generic_constraint_ids = pd.DataFrame({
    ...   'constraint_id': [1, 2],
    ...   'set': ['A', 'B']})

    >>> unit_bids_to_constraint_map = pd.DataFrame({
    ...   'variable_id': [0, 1],
    ...   'unit': ['X', 'Y'],
    ...   'service': ['energy', 'energy']})

    >>> lhs = create_unit_level_generic_constraint_lhs(generic_constraint_units, generic_constraint_ids,
    ...   unit_bids_to_constraint_map)

    >>> print(lhs)
       constraint_id  variable_id  coefficient
    0              1            0          0.9
    1              1            1          0.8

    Parameters
    ----------
    generic_constraint_units : pd.DataFrame

        =============  ==============================================================
        Columns:       Description:
        set            the unique identifier of the constraint set to map the
                       lhs coefficients to (as `str`)
        unit           the unit whose variables will be mapped to the lhs (as `str`)
        service        the service whose variables will be mapped to the lhs (as `str`)
        coefficient    the lhs coefficient (as `np.float64`)
        =============  ==============================================================

    generic_constraint_ids : pd.DataFrame

        =============  ===============================================================
        Columns:       Description:
        constraint_id  the unique identifier of the constraint (as `np.int64`)
        set            the constraint set that the id refers to (as `str`)
        =============  ===============================================================

    unit_bids_to_constraint_map : pd.DataFrame

        =============  =============================================================================
        Columns:       Description:
        variable_id    the id of the variable (as `np.int64`)
        unit           the unit level constraints the variable should map to (as `str`)
        service        the service type of the constraints the variables should map to (as `str`)
        =============  =============================================================================

    Returns
    -------
    lhs : pd.DataFrame

        =============  ===============================================================
        Columns:       Description:
        constraint_id  the unique identifier of the constraint (as `np.int64`)
        variable_id    the unique identifier of the variable (as `np.int64`)
        coefficient    the constraint level contribution to the lhs coefficient (as `np.float64`)
        =============  ===============================================================
    """
    unit_lhs = pd.merge(generic_constraint_units,
                        unit_bids_to_constraint_map.loc[:, ['unit', 'service', 'variable_id']],
                        on=['unit', 'service'])
    unit_lhs = pd.merge(unit_lhs, generic_constraint_ids.loc[:, ['constraint_id', 'set']], on='set')
    return unit_lhs.loc[:, ['constraint_id', 'variable_id', 'coefficient']]


def create_region_level_generic_constraint_lhs(generic_constraint_regions, generic_constraint_ids,
                                               regional_bids_to_constraint_map):
    """Find the lhs variables from regions for generic constraints.

    Examples
    --------

    >>> generic_constraint_regions = pd.DataFrame({
    ...   'set': ['A'],
    ...   'region': ['X'],
    ...   'service': ['energy'],
    ...   'coefficient': [0.9]})

    >>> generic_constraint_ids = pd.DataFrame({
    ...   'constraint_id': [1, 2],
    ...   'set': ['A', 'B']})

    >>> regional_bids_to_constraint_map = pd.DataFrame({
    ...   'variable_id': [0, 1],
    ...   'region': ['X', 'X'],
    ...   'service': ['energy', 'energy']})

    >>> lhs = create_region_level_generic_constraint_lhs(generic_constraint_regions, generic_constraint_ids,
    ...   regional_bids_to_constraint_map)

    >>> print(lhs)
       constraint_id  variable_id  coefficient
    0              1            0          0.9
    1              1            1          0.9

    Parameters
    ----------
    generic_constraint_regions : pd.DataFrame

        =============  ==============================================================
        Columns:       Description:
        set            the unique identifier of the constraint set to map the
                       lhs coefficients to (as `str`)
        region         the region whose variables will be mapped to the lhs (as `str`)
        service        the service whose variables will be mapped to the lhs (as `str`)
        coefficient    the lhs coefficient (as `np.float64`)
        =============  ==============================================================

    generic_constraint_ids : pd.DataFrame

        =============  ===============================================================
        Columns:       Description:
        constraint_id  the unique identifier of the constraint (as `np.int64`)
        set            the constraint set that the id refers to (as `str`)
        =============  ===============================================================

    regional_bids_to_constraint_map : pd.DataFrame

        =============  =============================================================================
        Columns:       Description:
        variable_id    the id of the variable (as `np.int64`)
        region         the region level constraints the variable should map to (as `str`)
        service        the service type of the constraints the variables should map to (as `str`)
        =============  =============================================================================

    Returns
    -------
    lhs : pd.DataFrame

        =============  ===============================================================
        Columns:       Description:
        constraint_id  the unique identifier of the constraint (as `np.int64`)
        variable_id    the unique identifier of the variable (as `np.int64`)
        coefficient    the constraint level contribution to the lhs coefficient (as `np.float64`)
        =============  ===============================================================
    """
    region_lhs = pd.merge(generic_constraint_regions,
                          regional_bids_to_constraint_map.loc[:, ['region', 'service', 'variable_id']],
                          on=['region', 'service'])
    region_lhs = pd.merge(region_lhs, generic_constraint_ids.loc[:, ['constraint_id', 'set']], on='set')
    return region_lhs.loc[:, ['constraint_id', 'variable_id', 'coefficient']]


def create_interconnector_generic_constraint_lhs(generic_constraint_interconnectors, generic_constraint_ids,
                                                 interconnector_variables):
    """Find the lhs variables from interconnectors for generic constraints.

    Examples
    --------

    >>> generic_constraint_interconnectors = pd.DataFrame({
    ...   'set': ['A'],
    ...   'interconnector': ['X'],
    ...   'coefficient': [0.9]})

    >>> generic_constraint_ids = pd.DataFrame({
    ...   'constraint_id': [1, 2],
    ...   'set': ['A', 'B']})

    >>> interconnector_variables = pd.DataFrame({
    ...   'variable_id': [0, 1],
    ...   'interconnector': ['X', 'X'],
    ...   'generic_constraint_factor': [1, 1]})

    >>> lhs = create_interconnector_generic_constraint_lhs(generic_constraint_interconnectors, generic_constraint_ids,
    ...   interconnector_variables)

    >>> print(lhs)
       constraint_id  variable_id  coefficient
    0              1            0          0.9
    1              1            1          0.9
    """
    interconnector_lhs = pd.merge(generic_constraint_interconnectors,
                                  interconnector_variables.loc[:, ['interconnector', 'variable_id',
                                                                   'generic_constraint_factor']],
                                  on=['interconnector'])
    interconnector_lhs = pd.merge(interconnector_lhs, generic_constraint_ids.loc[:, ['constraint_id', 'set']], on='set')
    interconnector_lhs['coefficient'] = interconnector_lhs['coefficient'] * interconnector_lhs[
        'generic_constraint_factor']
    return interconnector_lhs.loc[:, ['constraint_id', 'variable_id', 'coefficient']]

================
File: nempy\spot_markert_backend\unit_constraints.py
================
import pandas as pd
from nempy.help_functions import helper_functions as hf


def capacity(unit_limits, next_constraint_id):
    """Create the constraints that ensure the dispatch of a unit is capped by its capacity.

    A constraint of the following form will be created for each unit:

        bid 1 dispatched + bid 2 dispatched +. . .+ bid n dispatched <= capacity

    Examples
    --------

    >>> import pandas

    Defined the unit capacities.

    >>> unit_limits = pd.DataFrame({
    ...   'unit': ['A', 'B'],
    ...   'capacity': [100.0, 200.0]})

    >>> next_constraint_id = 0

    Create the constraint information.

    >>> type_and_rhs, variable_map = capacity(unit_limits, next_constraint_id)

    >>> print(type_and_rhs)
      unit service  constraint_id type    rhs
    0    A  energy              0   <=  100.0
    1    B  energy              1   <=  200.0

    >>> print(variable_map)
       constraint_id unit service  coefficient
    0              0    A  energy          1.0
    1              1    B  energy          1.0

    Parameters
    ----------
    unit_limits : pd.DataFrame
        Capacity by unit.

        ========  =====================================================================================
        Columns:  Description:
        unit      unique identifier of a dispatch unit (as `str`)
        capacity  The maximum output of the unit if unconstrained by ramp rate, in MW (as `np.float64`)
        ========  =====================================================================================

    next_constraint_id : int
        The next integer to start using for constraint ids.

    Returns
    -------
    type_and_rhs : pd.DataFrame
        The type and rhs of each constraint.

        =============  ===============================================================
        Columns:       Description:
        unit           unique identifier of a dispatch unit (as `str`)
        constraint_id  the id of the variable (as `int`)
        type           the type of the constraint, e.g. "=" (as `str`)
        rhs            the rhs of the constraint (as `np.float64`)
        =============  ===============================================================

    variable_map : pd.DataFrame
        The type of variables that should appear on the lhs of the constraint.

        =============  ==========================================================================
        Columns:       Description:
        constraint_id  the id of the constraint (as `np.int64`)
        unit           the unit variables the constraint should map too (as `str`)
        service        the service type of the variables the constraint should map to (as `str`)
        coefficient    the constraint factor in the lhs coefficient (as `np.float64`)
        =============  ==========================================================================
    """
    type_and_rhs, variable_map = create_constraints(unit_limits, next_constraint_id, 'capacity', '<=')
    return type_and_rhs, variable_map


def ramp_up(unit_limits, next_constraint_id, dispatch_interval):
    """Create the constraints that ensure the dispatch of a unit is capped by its ramp up rate.

    A constraint of the following form will be created for each unit:

        bid 1 dispatched + bid 2 dispatched + . + bid n dispatched <= initial_output + ramp_up_rate / dispatch_interval

    Examples
    --------

    >>> import pandas

    Defined the unit capacities.

    >>> unit_limits = pd.DataFrame({
    ...   'unit': ['A', 'B'],
    ...   'ramp_up_rate': [100.0, 200.0],
    ...   'initial_output': [50.0, 60.0]})

    >>> next_constraint_id = 0

    >>> dispatch_interval = 30

    Create the constraint information.

    >>> type_and_rhs, variable_map = ramp_up(unit_limits, next_constraint_id, dispatch_interval)

    >>> print(type_and_rhs)
      unit service  constraint_id type    rhs
    0    A  energy              0   <=  100.0
    1    B  energy              1   <=  160.0

    >>> print(variable_map)
       constraint_id unit service  coefficient
    0              0    A  energy          1.0
    1              1    B  energy          1.0

    Parameters
    ----------
    unit_limits : pd.DataFrame
        Ramp up rate and initial output by unit.

        ==============  =====================================================================================
        Columns:        Description:
        unit            unique identifier of a dispatch unit (as `str`)
        initial_output  the output of the unit at the start of the dispatch interval, in MW (as `np.float64`)
        ramp_up_rate    the maximum rate at which the unit can increase output, in MW/h (as `np.float64`)
        ==============  =====================================================================================

    next_constraint_id : int
        The next integer to start using for constraint ids.


    dispatch_interval : int
        The length of the dispatch interval in minutes.

    Returns
    -------
    type_and_rhs : pd.DataFrame
        The type and rhs of each constraint.

        =============  ===============================================================
        Columns:       Description:
        unit           unique identifier of a dispatch unit (as `str`)
        constraint_id  the id of the variable (as `int`)
        type           the type of the constraint, e.g. "=" (as `str`)
        rhs            the rhs of the constraint (as `np.float64`)
        =============  ===============================================================

    variable_map : pd.DataFrame
        The type of variables that should appear on the lhs of the constraint.

        =============  ==========================================================================
        Columns:       Description:
        constraint_id  the id of the constraint (as `np.int64`)
        unit           the unit variables the constraint should map too (as `str`)
        service        the service type of the variables the constraint should map to (as `str`)
        coefficient    the constraint factor in the lhs coefficient (as `np.float64`)
        =============  ==========================================================================
    """
    unit_limits['max_output'] = unit_limits['initial_output'] + unit_limits['ramp_up_rate'] * (dispatch_interval / 60)
    type_and_rhs, variable_map = create_constraints(unit_limits, next_constraint_id, 'max_output', '<=')
    return type_and_rhs, variable_map


def ramp_down(unit_limits, next_constraint_id, dispatch_interval):
    """Create the constraints that ensure the dispatch of a unit is limited by its ramp down rate.

    A constraint of the following form will be created for each unit:

        bid 1 dispatched + bid 2 dispatched + . + bid n dispatched >= initial_output - ramp_up_rate / dispatch_interval

    Examples
    --------

    >>> import pandas

    Defined the unit capacities.

    >>> unit_limits = pd.DataFrame({
    ...   'unit': ['A', 'B'],
    ...   'ramp_down_rate': [40.0, 20.0],
    ...   'initial_output': [50.0, 60.0]})

    >>> next_constraint_id = 0

    >>> dispatch_interval = 30

    Create the constraint information.

    >>> type_and_rhs, variable_map = ramp_down(unit_limits, next_constraint_id, dispatch_interval)

    >>> print(type_and_rhs)
      unit service  constraint_id type   rhs
    0    A  energy              0   >=  30.0
    1    B  energy              1   >=  50.0

    >>> print(variable_map)
       constraint_id unit service  coefficient
    0              0    A  energy          1.0
    1              1    B  energy          1.0

    Parameters
    ----------
    unit_limits : pd.DataFrame
        Ramp up rate and initial output by unit.

        ==============  =====================================================================================
        Columns:        Description:
        unit            unique identifier of a dispatch unit (as `str`)
        initial_output  the output of the unit at the start of the dispatch interval, in MW (as `np.float64`)
        ramp_down_rate  the maximum rate at which the unit can decrease output, in MW/h (as `np.float64`)
        ==============  =====================================================================================

    next_constraint_id : int
        The next integer to start using for constraint ids.


    dispatch_interval : int
        The length of the dispatch interval in minutes.

    Returns
    -------
    type_and_rhs : pd.DataFrame
        The type and rhs of each constraint.

        =============  ===============================================================
        Columns:       Description:
        unit           unique identifier of a dispatch unit (as `str`)
        constraint_id  the id of the variable (as `int`)
        type           the type of the constraint, e.g. "=" (as `str`)
        rhs            the rhs of the constraint (as `np.float64`)
        =============  ===============================================================

    variable_map : pd.DataFrame
        The type of variables that should appear on the lhs of the constraint.

        =============  ==========================================================================
        Columns:       Description:
        constraint_id  the id of the constraint (as `np.int64`)
        unit           the unit variables the constraint should map too (as `str`)
        service        the service type of the variables the constraint should map to (as `str`)
        coefficient    the constraint factor in the lhs coefficient (as `np.float64`)
        =============  ==========================================================================
    """
    unit_limits['min_output'] = unit_limits['initial_output'] - unit_limits['ramp_down_rate'] * (dispatch_interval / 60)
    type_and_rhs, variable_map = create_constraints(unit_limits, next_constraint_id, 'min_output', '>=')
    return type_and_rhs, variable_map


def fcas_max_availability(fcas_availability, next_constraint_id):
    """Create the constraints that ensure the dispatch of a unit fcas is capped by its availability.

    A constraint of the following form will be created for each unit:

        bid 1 dispatched + bid 2 dispatched +. . .+ bid n dispatched <= availability

    Examples
    --------

    >>> import pandas

    Defined the unit fcas availability.

    >>> fcas_availability = pd.DataFrame({
    ...   'unit': ['A', 'B'],
    ...   'service': ['raise_reg', 'lower_6s'],
    ...   'max_availability': [100.0, 200.0]})

    >>> next_constraint_id = 0

    Create the constraint information.

    >>> type_and_rhs, variable_map = fcas_max_availability(fcas_availability, next_constraint_id)

    >>> print(type_and_rhs)
      unit    service  constraint_id type    rhs
    0    A  raise_reg              0   <=  100.0
    1    B   lower_6s              1   <=  200.0

    >>> print(variable_map)
       constraint_id unit    service  coefficient
    0              0    A  raise_reg          1.0
    1              1    B   lower_6s          1.0

    Parameters
    ----------
    fcas_availability : pd.DataFrame
        Availability by unit and service.

        ================   ======================================================================
        Columns:           Description:
        unit               unique identifier of a dispatch unit (as `str`)
        service            the fcas service being offered (as `str`)
        max_availability   the maximum volume of the fcas service in MW (as `np.float64`)
        ================   ======================================================================

    next_constraint_id : int
        The next integer to start using for constraint ids.

    Returns
    -------
    type_and_rhs : pd.DataFrame
        The type and rhs of each constraint.

        =============  ===============================================================
        Columns:       Description:
        unit           unique identifier of a dispatch unit (as `str`)
        constraint_id  the id of the variable (as `int`)
        type           the type of the constraint, e.g. "=" (as `str`)
        rhs            the rhs of the constraint (as `np.float64`)
        =============  ===============================================================

    variable_map : pd.DataFrame
        The type of variables that should appear on the lhs of the constraint.

        =============  ==========================================================================
        Columns:       Description:
        constraint_id  the id of the constraint (as `np.int64`)
        unit           the unit variables the constraint should map too (as `str`)
        service        the service type of the variables the constraint should map to (as `str`)
        coefficient    the constraint factor in the lhs coefficient (as `np.float64`)
        =============  ==========================================================================
    """
    type_and_rhs, variable_map = create_constraints(fcas_availability, next_constraint_id, 'max_availability', '<=')
    return type_and_rhs, variable_map


def create_fast_start_profile_constraints(fast_start_profiles, next_constraint_id, dispatch_interval):

    mode_one_cons = fast_start_mode_one_constraints(fast_start_profiles)
    mode_two_cons = fast_start_mode_two_constraints(fast_start_profiles)
    mode_three_cons = fast_start_mode_three_constraints(fast_start_profiles)
    mode_four_cons = fast_start_mode_four_constraints(fast_start_profiles)

    type_and_rhs = []
    variable_map = []

    if not mode_one_cons.empty:
        mode_one_max_type_rhs, mode_one_max_variable_map = \
            create_constraints(mode_one_cons, next_constraint_id, 'max', '<=')
        next_constraint_id = mode_one_max_type_rhs['constraint_id'].max() + 1
        type_and_rhs.append(mode_one_max_type_rhs)
        variable_map.append(mode_one_max_variable_map)

    if not mode_two_cons.empty:
        mode_two_min_type_rhs, mode_two_min_variable_map = \
            create_constraints(mode_two_cons, next_constraint_id, 'min', '>=')
        next_constraint_id = mode_two_min_type_rhs['constraint_id'].max() + 1
        mode_two_max_type_rhs, mode_two_max_variable_map = \
            create_constraints(mode_two_cons, next_constraint_id, 'max', '<=')
        next_constraint_id = mode_two_max_type_rhs['constraint_id'].max() + 1
        type_and_rhs.append(mode_two_min_type_rhs)
        type_and_rhs.append(mode_two_max_type_rhs)
        variable_map.append(mode_two_max_variable_map)
        variable_map.append(mode_two_min_variable_map)

    if not mode_three_cons.empty:
        mode_three_min_type_rhs, mode_three_min_variable_map = \
            create_constraints(mode_three_cons, next_constraint_id, 'min', '>=')
        next_constraint_id = mode_three_min_type_rhs['constraint_id'].max() + 1
        type_and_rhs.append(mode_three_min_type_rhs)
        variable_map.append(mode_three_min_variable_map)

    if not mode_four_cons.empty:
        mode_four_min_type_rhs, mode_four_min_variable_map = \
            create_constraints(mode_four_cons, next_constraint_id, 'min', '>=')
        type_and_rhs.append(mode_four_min_type_rhs)
        variable_map.append(mode_four_min_variable_map)

    if len(type_and_rhs) > 0:
        type_and_rhs = pd.concat(type_and_rhs)
        variable_map = pd.concat(variable_map)
    else:
        type_and_rhs = pd.DataFrame()
        variable_map = pd.DataFrame()

    return type_and_rhs, variable_map


def create_constraints(unit_limits, next_constraint_id, rhs_col, direction):
    # If no service column is present assume the constraints are for the energy service.
    if 'service' not in unit_limits.columns:
        unit_limits['service'] = 'energy'

    # Create a constraint for each unit in unit limits.
    type_and_rhs = hf.save_index(unit_limits.reset_index(drop=True), 'constraint_id', next_constraint_id)
    type_and_rhs = type_and_rhs.loc[:, ['unit', 'service', 'constraint_id', rhs_col]]
    type_and_rhs['type'] = direction  # the type i.e. >=, <=, or = is set by a parameter.
    type_and_rhs['rhs'] = type_and_rhs[rhs_col]  # column used to set the rhs is set by a parameter.
    type_and_rhs = type_and_rhs.loc[:, ['unit', 'service', 'constraint_id', 'type', 'rhs']]

    # These constraints always map to energy variables and have a coefficient of one.
    variable_map = type_and_rhs.loc[:, ['constraint_id', 'unit', 'service']]
    variable_map['coefficient'] = 1.0

    return type_and_rhs, variable_map


def fast_start_mode_one_constraints(fast_start_profile):
    units_ending_in_mode_one = fast_start_profile[(fast_start_profile['end_mode'].isin([0, 1]))].copy()
    units_ending_in_mode_one['max'] = 0.0
    units_ending_in_mode_one['min'] = 0.0
    units_ending_in_mode_one = units_ending_in_mode_one.loc[:, ['unit', 'min', 'max']]
    return units_ending_in_mode_one


def fast_start_mode_two_constraints(fast_start_profile):
    units_ending_in_mode_two = fast_start_profile[(fast_start_profile['end_mode'] == 2)].copy()
    units_ending_in_mode_two['target'] = (((units_ending_in_mode_two['time_in_end_mode'])
                                           / units_ending_in_mode_two['mode_two_length']) *
                                          units_ending_in_mode_two['min_loading'])
    units_ending_in_mode_two['min'] = units_ending_in_mode_two['target']
    units_ending_in_mode_two['max'] = units_ending_in_mode_two['target']
    units_ending_in_mode_two = units_ending_in_mode_two.loc[:, ['unit', 'min', 'max']]
    return units_ending_in_mode_two


def fast_start_mode_three_constraints(fast_start_profile):
    units_ending_in_mode_three = fast_start_profile[(fast_start_profile['end_mode'] == 3)].copy()
    units_ending_in_mode_three['min'] = units_ending_in_mode_three['min_loading']
    units_ending_in_mode_three = units_ending_in_mode_three.loc[:, ['unit', 'min']]
    return units_ending_in_mode_three


def fast_start_mode_four_constraints(fast_start_profile):
    units_ending_in_mode_four = fast_start_profile[fast_start_profile['end_mode'] == 4].copy()
    units_ending_in_mode_four['target'] = (units_ending_in_mode_four['min_loading'] -
                                           (((units_ending_in_mode_four['time_in_end_mode']) /
                                             units_ending_in_mode_four['mode_four_length']) *
                                            units_ending_in_mode_four['min_loading']))
    units_ending_in_mode_four['min'] = units_ending_in_mode_four['target']
    units_ending_in_mode_four['max'] = units_ending_in_mode_four['target']
    units_ending_in_mode_four = units_ending_in_mode_four.loc[:, ['unit', 'min', 'max']]
    return units_ending_in_mode_four


def tie_break_constraints(price_bids, bid_decision_variables, unit_regions, next_constraint_id):
    energy_price_bids = price_bids[price_bids['service'] == 'energy']
    energy_price_bids = pd.merge(energy_price_bids,
                                 bid_decision_variables.loc[:, ['variable_id', 'upper_bound']],
                                 on='variable_id')
    energy_price_bids = pd.merge(energy_price_bids, unit_regions.loc[:, ['unit', 'region']], on='unit')

    constraints = pd.merge(energy_price_bids, energy_price_bids, on=['cost', 'region'])
    constraints = constraints[constraints['unit_x'] != constraints['unit_y']]

    def make_id(unit_x, band_x, unit_y, band_y):
        name = sorted([unit_x, str(band_x), unit_y, str(band_y)])
        name = ''.join(name)
        return name

    constraints['name'] = \
        constraints.apply(lambda x: make_id(x['unit_x'], x['capacity_band_x'],
                                            x['unit_y'], x['capacity_band_y']), axis=1)

    constraints = constraints.drop_duplicates('name')

    constraints = constraints.loc[:, ['variable_id_x', 'upper_bound_x', 'variable_id_y', 'upper_bound_y']]
    constraints = hf.save_index(constraints, 'constraint_id', next_constraint_id)

    lhs_one = constraints.loc[:, ['constraint_id', 'variable_id_x', 'upper_bound_x']]
    lhs_one['variable_id'] = lhs_one['variable_id_x']
    lhs_one['coefficient'] = 1 / lhs_one['upper_bound_x']

    lhs_two = constraints.loc[:, ['constraint_id', 'variable_id_y', 'upper_bound_y']]
    lhs_two['variable_id'] = lhs_two['variable_id_y']
    lhs_two['coefficient'] = - 1 / lhs_two['upper_bound_y']

    lhs = pd.concat([lhs_one.loc[:, ['constraint_id', 'variable_id', 'coefficient']],
                     lhs_two.loc[:, ['constraint_id', 'variable_id', 'coefficient']]])

    rhs = constraints.loc[:, ['constraint_id']]

    rhs['type'] = '='
    rhs['rhs'] = 0.0
    return lhs, rhs

================
File: nempy\spot_markert_backend\variable_ids.py
================
import pandas as pd
import numpy as np
from nempy.help_functions import helper_functions as hf


def bids(volume_bids, unit_info, next_variable_id):
    """Create decision variables that correspond to unit bids, for use in the linear program.

    This function defines the needed parameters for each variable, with a lower bound equal to zero, an upper bound
    equal to the bid volume, and a variable type of continuous. There is no limit on the number of bid bands and each
    column in the capacity_bids DataFrame other than unit is treated as a bid band. Volume bids should be positive.
    numeric values only.

    Examples
    --------

    >>> import pandas

    A set of capacity bids.

    >>> volume_bids = pd.DataFrame({
    ...   'unit': ['A', 'B'],
    ...   '1': [10.0, 50.0],
    ...   '2': [20.0, 30.0]})

    The locations of the units.

    >>> unit_info = pd.DataFrame({
    ...   'unit': ['A', 'B'],
    ...   'region': ['NSW', 'X'],
    ...   'dispatch_type': ['generator', 'load']})

    >>> next_variable_id = 0

    Create the decision variables and their mapping into constraints.

    >>> decision_variables, unit_level_constraint_map, regional_constraint_map = bids(
    ...   volume_bids, unit_info, next_variable_id)

    >>> print(decision_variables)
      unit capacity_band service  variable_id  lower_bound  upper_bound        type
    0    A             1  energy            0          0.0         10.0  continuous
    1    A             2  energy            1          0.0         20.0  continuous
    2    B             1  energy            2          0.0         50.0  continuous
    3    B             2  energy            3          0.0         30.0  continuous

    >>> print(unit_level_constraint_map)
       variable_id unit service  coefficient
    0            0    A  energy          1.0
    1            1    A  energy          1.0
    2            2    B  energy          1.0
    3            3    B  energy          1.0

    >>> print(regional_constraint_map)
       variable_id region service  coefficient
    0            0    NSW  energy          1.0
    1            1    NSW  energy          1.0
    2            2      X  energy         -1.0
    3            3      X  energy         -1.0

    Parameters
    ----------
    volume_bids : pd.DataFrame
        Bids by unit, in MW, can contain up to n bid bands.

        ========  ===============================================================
        Columns:  Description:
        unit      unique identifier of a dispatch unit (as `str`)
        service   the service being provided, optional, if missing energy assumed
                  (as `str`)
        1         bid volume in the 1st band, in MW (as `float`)
        2         bid volume in the 2nd band, in MW (as `float`)
        n         bid volume in the nth band, in MW (as `float`)
        ========  ===============================================================

    unit_info : pd.DataFrame
        The region each unit is located in.

        ========  ======================================================
        Columns:  Description:
        unit      unique identifier of a dispatch unit (as `str`)
        region    unique identifier of a market region (as `str`)
        ========  ======================================================

    next_variable_id : int
        The next integer to start using for variables ids.

    Returns
    -------
    decision_variables : pd.DataFrame

        =============  ===============================================================
        Columns:       Description:
        unit           unique identifier of a dispatch unit (as `str`)
        capacity_band  the bid band of the variable (as `str`)
        variable_id    the id of the variable (as `int`)
        lower_bound    the lower bound of the variable, is zero for bids (as `np.float64`)
        upper_bound    the upper bound of the variable, the volume bid (as `np.float64`)
        type           the type of variable, is continuous for bids  (as `str`)
        =============  ===============================================================

    unit_level_constraint_map : pd.DataFrame

        =============  =============================================================================
        Columns:       Description:
        variable_id    the id of the variable (as `np.int64`)
        unit           the unit level constraints the variable should map to (as `str`)
        service        the service type of the constraints the variables should map to (as `str`)
        coefficient    the upper bound of the variable, the volume bid (as `np.float64`)
        =============  =============================================================================

    regional_constraint_map : pd.DataFrame

        =============  =============================================================================
        Columns:       Description:
        variable_id    the id of the variable (as `np.int64`)
        region         the regional constraints the variable should map to (as `str`)
        service        the service type of the constraints the variables should map to (as `str`)
        coefficient    the upper bound of the variable, the volume bid (as `np.float64`)
        =============  =============================================================================
    """
    # If no service column is provided assume bids are for energy.
    if 'service' not in volume_bids.columns:
        volume_bids['service'] = 'energy'

    # Get a list of all the columns that contain volume bids.
    bid_bands = [col for col in volume_bids.columns if col not in ['unit', 'service']]
    # Reshape the table so each bid band is on it own row.
    decision_variables = hf.stack_columns(volume_bids, cols_to_keep=['unit', 'service'],
                                          cols_to_stack=bid_bands, type_name='capacity_band', value_name='upper_bound')
    decision_variables = decision_variables[decision_variables['upper_bound'] >= 0.0001]
    # Group units together in the decision variable table.
    decision_variables = decision_variables.sort_values(['unit', 'capacity_band'])
    # Create a unique identifier for each decision variable.
    decision_variables = hf.save_index(decision_variables, 'variable_id', next_variable_id)
    # The lower bound of bidding decision variables will always be zero.
    decision_variables['lower_bound'] = 0.0
    decision_variables['type'] = 'continuous'

    constraint_map = decision_variables.loc[:, ['variable_id', 'unit', 'service']]
    constraint_map = pd.merge(constraint_map, unit_info.loc[:, ['unit', 'region', 'dispatch_type']], 'inner', on='unit')
    regional_constraint_map = constraint_map.loc[:,  ['variable_id', 'region', 'service', 'dispatch_type']]
    regional_constraint_map['coefficient'] = np.where((regional_constraint_map['dispatch_type'] == 'load') &
                                                      (regional_constraint_map['service'] == 'energy'), -1.0, 1.0)
    regional_constraint_map = regional_constraint_map.drop('dispatch_type', axis=1)
    unit_level_constraint_map = constraint_map.loc[:,  ['variable_id', 'unit', 'service']]
    unit_level_constraint_map['coefficient'] = 1.0

    decision_variables = \
        decision_variables.loc[:, ['unit', 'capacity_band', 'service', 'variable_id', 'lower_bound', 'upper_bound',
                                   'type']]

    return decision_variables, unit_level_constraint_map, regional_constraint_map

================
File: publications\all_features_example.py
================
# Notice:
# - This script downloads large volumes of historical market data from AEMO's nemweb
#   portal. The boolean on line 20 can be changed to prevent this happening repeatedly
#   once the data has been downloaded.

import sqlite3
import pandas as pd
import random
from datetime import datetime, timedelta

from nempy import markets
from nempy.historical_inputs import loaders, mms_db, \
    xml_cache, units, demand, interconnectors, \
    constraints

# The size of historical data files for a full year of 5 min dispatch
# is very large, approximately 800 GB, for this reason the data is
# stored on an external SSD.
con = sqlite3.connect('F:/nempy_test_files/historical_mms.db')
mms_db_manager = mms_db.DBManager(connection=con)
xml_cache_manager = xml_cache.XMLCacheManager('F:/nempy_test_files/nemde_cache')

# The second time this example is run on a machine this flag can
# be set to false to save downloading the data again.
download_inputs = False

if download_inputs:
    mms_db_manager.populate(start_year=2019, start_month=1,
                            end_year=2019, end_month=12)
    xml_cache_manager.populate(start_year=2019, start_month=1,
                               end_year=2020, end_month=1)

raw_inputs_loader = loaders.RawInputsLoader(
    nemde_xml_cache_manager=xml_cache_manager,
    market_management_system_database=mms_db_manager)


# Define a function for creating a list of randomly selected dispatch
# intervals
def get_test_intervals(number):
    start_time = datetime(year=2019, month=1, day=1, hour=0, minute=0)
    end_time = datetime(year=2019, month=12, day=31, hour=0, minute=0)
    difference = end_time - start_time
    difference_in_5_min_intervals = difference.days * 12 * 24
    random.seed(1)
    intervals = random.sample(range(1, difference_in_5_min_intervals), number)
    times = [start_time + timedelta(minutes=5 * i) for i in intervals]
    times_formatted = [t.isoformat().replace('T', ' ').replace('-', '/') for t in times]
    return times_formatted


# List for saving outputs to.
outputs = []

# Create and dispatch the spot market for each dispatch interval.
for interval in get_test_intervals(number=1000):
    raw_inputs_loader.set_interval(interval)
    unit_inputs = units.UnitData(raw_inputs_loader)
    interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
    constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
    demand_inputs = demand.DemandData(raw_inputs_loader)

    unit_info = unit_inputs.get_unit_info()
    market = markets.SpotMarket(market_regions=['QLD1', 'NSW1', 'VIC1',
                                                'SA1', 'TAS1'],
                                unit_info=unit_info)

    # By default the CBC open source solver is used, but GUROBI is
    # also supported
    market.solver_name = 'CBC'  # or could be 'GUROBI'

    # Set bids
    volume_bids, price_bids = unit_inputs.get_processed_bids()
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_price_bids(price_bids)

    # Set bid in capacity limits
    unit_bid_limit = unit_inputs.get_unit_bid_availability()
    market.set_unit_bid_capacity_constraints(unit_bid_limit)
    cost = constraint_inputs.get_constraint_violation_prices()['unit_capacity']
    market.make_constraints_elastic('unit_bid_capacity', violation_cost=cost)

    # Set limits provided by the unconstrained intermittent generation
    # forecasts. Primarily for wind and solar.
    unit_uigf_limit = unit_inputs.get_unit_uigf_limits()
    market.set_unconstrained_intermitent_generation_forecast_constraint(
        unit_uigf_limit)
    cost = constraint_inputs.get_constraint_violation_prices()['uigf']
    market.make_constraints_elastic('uigf_capacity', violation_cost=cost)

    # Set unit ramp rates.
    ramp_rates = unit_inputs.get_ramp_rates_used_for_energy_dispatch()
    market.set_unit_ramp_up_constraints(
        ramp_rates.loc[:, ['unit', 'initial_output', 'ramp_up_rate']])
    market.set_unit_ramp_down_constraints(
        ramp_rates.loc[:, ['unit', 'initial_output', 'ramp_down_rate']])
    cost = constraint_inputs.get_constraint_violation_prices()['ramp_rate']
    market.make_constraints_elastic('ramp_up', violation_cost=cost)
    market.make_constraints_elastic('ramp_down', violation_cost=cost)

    # Set unit FCAS trapezium constraints.
    unit_inputs.add_fcas_trapezium_constraints()
    cost = constraint_inputs.get_constraint_violation_prices()['fcas_max_avail']
    fcas_availability = unit_inputs.get_fcas_max_availability()
    market.set_fcas_max_availability(fcas_availability)
    market.make_constraints_elastic('fcas_max_availability', cost)
    cost = constraint_inputs.get_constraint_violation_prices()['fcas_profile']
    regulation_trapeziums = unit_inputs.get_fcas_regulation_trapeziums()
    market.set_energy_and_regulation_capacity_constraints(regulation_trapeziums)
    market.make_constraints_elastic('energy_and_regulation_capacity', cost)
    scada_ramp_down_rates = unit_inputs.get_scada_ramp_down_rates_of_lower_reg_units()
    market.set_joint_ramping_constraints_lower_reg(scada_ramp_down_rates)
    market.make_constraints_elastic('joint_ramping_lower_reg', cost)
    scada_ramp_up_rates = unit_inputs.get_scada_ramp_up_rates_of_raise_reg_units()
    market.set_joint_ramping_constraints_raise_reg(scada_ramp_up_rates)
    market.make_constraints_elastic('joint_ramping_raise_reg', cost)
    contingency_trapeziums = unit_inputs.get_contingency_services()
    market.set_joint_capacity_constraints(contingency_trapeziums)
    market.make_constraints_elastic('joint_capacity', cost)

    # Set interconnector definitions, limits and loss models.
    interconnectors_definitions = \
        interconnector_inputs.get_interconnector_definitions()
    loss_functions, interpolation_break_points = \
        interconnector_inputs.get_interconnector_loss_model()
    market.set_interconnectors(interconnectors_definitions)
    market.set_interconnector_losses(loss_functions,
                                     interpolation_break_points)

    # Add generic constraints and FCAS market constraints.
    fcas_requirements = constraint_inputs.get_fcas_requirements()
    market.set_fcas_requirements_constraints(fcas_requirements)
    violation_costs = constraint_inputs.get_violation_costs()
    market.make_constraints_elastic('fcas', violation_cost=violation_costs)
    generic_rhs = constraint_inputs.get_rhs_and_type_excluding_regional_fcas_constraints()
    market.set_generic_constraints(generic_rhs)
    market.make_constraints_elastic('generic', violation_cost=violation_costs)
    unit_generic_lhs = constraint_inputs.get_unit_lhs()
    market.link_units_to_generic_constraints(unit_generic_lhs)
    interconnector_generic_lhs = constraint_inputs.get_interconnector_lhs()
    market.link_interconnectors_to_generic_constraints(
        interconnector_generic_lhs)

    # Set the operational demand to be met by dispatch.
    regional_demand = demand_inputs.get_operational_demand()
    market.set_demand_constraints(regional_demand)
    # Get unit dispatch without fast start constraints and use it to
    # make fast start unit commitment decisions.
    market.dispatch()
    dispatch = market.get_unit_dispatch()
    fast_start_profiles = unit_inputs.get_fast_start_profiles_for_dispatch(dispatch)
    market.set_fast_start_constraints(fast_start_profiles)
    if 'fast_start' in market.get_constraint_set_names():
        cost = constraint_inputs.get_constraint_violation_prices()['fast_start']
        market.make_constraints_elastic('fast_start', violation_cost=cost)

    # If AEMO historical used the over constrained dispatch rerun
    # process then allow it to be used in dispatch. This is needed
    # because sometimes the conditions for over constrained dispatch
    # are present but the rerun process isn't used.
    if constraint_inputs.is_over_constrained_dispatch_rerun():
        market.dispatch(allow_over_constrained_dispatch_re_run=True,
                        energy_market_floor_price=-1000.0,
                        energy_market_ceiling_price=14500.0,
                        fcas_market_ceiling_price=1000.0)
    else:
        # The market price ceiling and floor are not needed here
        # because they are only used for the over constrained
        # dispatch rerun process.
        market.dispatch(allow_over_constrained_dispatch_re_run=False)

    # Save prices from this interval
    prices = market.get_energy_prices()
    prices['time'] = interval

    # Getting historical prices for comparison. Note, ROP price, which is
    # the regional reference node price before the application of any
    # price scaling by AEMO, is used for comparison.
    historical_prices = mms_db_manager.DISPATCHPRICE.get_data(interval)

    prices = pd.merge(prices, historical_prices,
                      left_on=['time', 'region'],
                      right_on=['SETTLEMENTDATE', 'REGIONID'])

    outputs.append(
        prices.loc[:, ['time', 'region', 'price',
                       'SETTLEMENTDATE', 'REGIONID', 'ROP']])

con.close()
outputs = pd.concat(outputs)
outputs = outputs.sort_values('ROP')
outputs = outputs.reset_index(drop=True)
outputs.to_csv('energy_price_results_2019_1000_intervals.csv')

================
File: publications\energy_only_market.py
================
# Notice:
# - This script downloads large volumes of historical market data from AEMO's nemweb
#   portal. The boolean on line 20 can be changed to prevent this happening repeatedly
#   once the data has been downloaded.

import sqlite3
import pandas as pd
import random
from datetime import datetime, timedelta

from nempy import markets
from nempy.historical_inputs import loaders, mms_db, \
    xml_cache, units, demand, interconnectors, \
    constraints

# The size of historical data files for a full year of 5 min dispatch
# is very large, approximately 800 GB, for this reason the data is
# stored on an external SSD.
con = sqlite3.connect('historical_mms.db')
mms_db_manager = mms_db.DBManager(connection=con)
xml_cache_manager = xml_cache.XMLCacheManager('nemde_cache')

# The second time this example is run on a machine this flag can
# be set to false to save downloading the data again.
download_inputs = True

if download_inputs:
    # This requires approximately 5 GB of storage.
    mms_db_manager.populate(start_year=2019, start_month=1,
                            end_year=2019, end_month=12)

    # This requires approximately 3.5 GB of storage.
    xml_cache_manager.populate(start_year=2019, start_month=1, start_day=1,
                               end_year=2020, end_month=1, end_day=1)

raw_inputs_loader = loaders.RawInputsLoader(
    nemde_xml_cache_manager=xml_cache_manager,
    market_management_system_database=mms_db_manager)


# Define a function for creating a list of randomly selected dispatch
# intervals
def get_test_intervals(number):
    start_time = datetime(year=2019, month=1, day=1, hour=0, minute=0)
    end_time = datetime(year=2019, month=12, day=31, hour=0, minute=0)
    difference = end_time - start_time
    difference_in_5_min_intervals = difference.days * 12 * 24
    random.seed(1)
    intervals = random.sample(range(1, difference_in_5_min_intervals), number)
    times = [start_time + timedelta(minutes=5 * i) for i in intervals]
    times_formatted = [t.isoformat().replace('T', ' ').replace('-', '/') for t in times]
    return times_formatted


# List for saving outputs to.
outputs = []

# Create and dispatch the spot market for each dispatch interval.
for interval in get_test_intervals(number=1000):
    raw_inputs_loader.set_interval(interval)
    unit_inputs = units.UnitData(raw_inputs_loader)
    interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
    constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
    demand_inputs = demand.DemandData(raw_inputs_loader)

    unit_info = unit_inputs.get_unit_info()
    market = markets.SpotMarket(market_regions=['QLD1', 'NSW1', 'VIC1',
                                                'SA1', 'TAS1'],
                                unit_info=unit_info)

    # By default the CBC open source solver is used, but GUROBI is
    # also supported
    market.solver_name = 'CBC'  # or could be 'GUROBI'

    # Set bids
    volume_bids, price_bids = unit_inputs.get_processed_bids()
    volume_bids = volume_bids[volume_bids['service'] == 'energy']
    price_bids = price_bids[price_bids['service'] == 'energy']
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_price_bids(price_bids)

    # Set bid in capacity limits
    unit_bid_limit = unit_inputs.get_unit_bid_availability()
    market.set_unit_bid_capacity_constraints(unit_bid_limit)
    cost = constraint_inputs.get_constraint_violation_prices()['unit_capacity']
    market.make_constraints_elastic('unit_bid_capacity', violation_cost=cost)

    # Set limits provided by the unconstrained intermittent generation
    # forecasts. Primarily for wind and solar.
    unit_uigf_limit = unit_inputs.get_unit_uigf_limits()
    market.set_unconstrained_intermitent_generation_forecast_constraint(
        unit_uigf_limit)
    cost = constraint_inputs.get_constraint_violation_prices()['uigf']
    market.make_constraints_elastic('uigf_capacity', violation_cost=cost)

    # Set unit ramp rates.
    ramp_rates = unit_inputs.get_ramp_rates_used_for_energy_dispatch()
    market.set_unit_ramp_up_constraints(
        ramp_rates.loc[:, ['unit', 'initial_output', 'ramp_up_rate']])
    market.set_unit_ramp_down_constraints(
        ramp_rates.loc[:, ['unit', 'initial_output', 'ramp_down_rate']])
    cost = constraint_inputs.get_constraint_violation_prices()['ramp_rate']
    market.make_constraints_elastic('ramp_up', violation_cost=cost)
    market.make_constraints_elastic('ramp_down', violation_cost=cost)

    # Set interconnector definitions, limits and loss models.
    interconnectors_definitions = \
        interconnector_inputs.get_interconnector_definitions()
    loss_functions, interpolation_break_points = \
        interconnector_inputs.get_interconnector_loss_model()
    market.set_interconnectors(interconnectors_definitions)
    market.set_interconnector_losses(loss_functions,
                                     interpolation_break_points)

    # Set the operational demand to be met by dispatch.
    regional_demand = demand_inputs.get_operational_demand()
    market.set_demand_constraints(regional_demand)
    market.dispatch()

    # Save prices from this interval
    prices = market.get_energy_prices()
    prices['time'] = interval

    # Getting historical prices for comparison. Note, ROP price, which is
    # the regional reference node price before the application of any
    # price scaling by AEMO, is used for comparison.
    historical_prices = mms_db_manager.DISPATCHPRICE.get_data(interval)

    prices = pd.merge(prices, historical_prices,
                      left_on=['time', 'region'],
                      right_on=['SETTLEMENTDATE', 'REGIONID'])

    outputs.append(
        prices.loc[:, ['time', 'region', 'price',
                       'SETTLEMENTDATE', 'REGIONID', 'ROP']])

con.close()
outputs = pd.concat(outputs)
outputs = outputs.sort_values('ROP')
outputs = outputs.reset_index(drop=True)
outputs.to_csv('energy_price_results_2019_1000_intervals_without_FCAS_or_generic_constraints.csv')

================
File: tests\build_historical_test_data_cache_and_db.py
================
import sqlite3
import pickle
from nempy.historical_inputs import mms_db, xml_cache


running_for_first_time = True

con = sqlite3.connect('D:/nempy_test_files/historical_mms.db')
mms_db_manager = mms_db.DBManager(connection=con)

xml_cache_manager = xml_cache.XMLCacheManager('D:/nempy_test_files/nemde_cache')

if running_for_first_time:
    mms_db_manager.populate(start_year=2019, start_month=1, end_year=2019, end_month=12)
    #xml_cache_manager.populate(start_year=2019, start_month=1, end_year=2019, end_month=12)

get_violation_intervals = False

if get_violation_intervals:
    interval_with_fast_start_violations = \
        xml_cache_manager.find_intervals_with_violations(limit=100000, start_year=2019, start_month=1,
                                                         end_year=2019, end_month=12)

    with open('interval_with_violations.pickle', 'wb') as f:
        pickle.dump(interval_with_fast_start_violations, f, pickle.HIGHEST_PROTOCOL)

con.close()

================
File: tests\get_violation_times.py
================
import pickle

from nempy.historical_inputs import xml_cache

import os
cwd = os.getcwd()

print(cwd)

historical_inputs = xml_cache.XMLCacheManager('test_files/historical_xml_files')

interval_with_violations = \
    historical_inputs.find_intervals_with_violations(limit=1000000,
                                                     start_year=2019, start_month=2,
                                                     end_year=2019, end_month=2)

with open('interval_with_violations.pickle', 'wb') as f:
    pickle.dump(interval_with_violations, f, pickle.HIGHEST_PROTOCOL)

================
File: tests\historical_market_builder.py
================
import pandas as pd
import pytest
import numpy as np

from nempy import markets
from nempy.help_functions import helper_functions as hf
from nempy.historical_inputs import mms_db as hi, demand


class SpotMarketBuilder:
    def __init__(self, unit_inputs, interconnector_inputs, constraint_inputs, demand_inputs):

        self.unit_inputs = unit_inputs
        self.interconnector_inputs = interconnector_inputs
        self.constraint_inputs = constraint_inputs
        self.regional_demand_inputs = demand_inputs

        unit_info = self.unit_inputs.get_unit_info()
        self.market = markets.SpotMarket(market_regions=['QLD1', 'NSW1', 'VIC1', 'SA1', 'TAS1'], unit_info=unit_info)
        self.market.solver_name = 'CBC'

    def set_solver(self, solver_name):
        self.market.solver_name = solver_name

    def add_unit_bids_to_market(self):
        volume_bids, price_bids = self.unit_inputs.get_processed_bids()
        self.market.set_unit_volume_bids(volume_bids)
        self.market.set_unit_price_bids(price_bids)

    def set_unit_limit_constraints(self):
        unit_bid_limit = self.unit_inputs.get_unit_bid_availability()
        self.market.set_unit_bid_capacity_constraints(unit_bid_limit)
        cost = self.constraint_inputs.get_constraint_violation_prices()['unit_capacity']
        self.market.make_constraints_elastic('unit_bid_capacity', violation_cost=cost)
        unit_uigf_limit = self.unit_inputs.get_unit_uigf_limits()
        self.market.set_unconstrained_intermitent_generation_forecast_constraint(unit_uigf_limit)
        cost = self.constraint_inputs.get_constraint_violation_prices()['uigf']
        self.market.make_constraints_elastic('uigf_capacity', violation_cost=cost)

    def set_ramp_rate_limits(self):
        ramp_rates = self.unit_inputs.get_ramp_rates_used_for_energy_dispatch()
        self.market.set_unit_ramp_up_constraints(
            ramp_rates.loc[:, ['unit', 'initial_output', 'ramp_up_rate']])
        self.market.set_unit_ramp_down_constraints(
            ramp_rates.loc[:, ['unit', 'initial_output', 'ramp_down_rate']])
        cost = self.constraint_inputs.get_constraint_violation_prices()['ramp_rate']
        self.market.make_constraints_elastic('ramp_up', violation_cost=cost)
        self.market.make_constraints_elastic('ramp_down', violation_cost=cost)

    def set_fast_start_constraints(self):
        self.market.dispatch()
        dispatch = self.market.get_unit_dispatch()
        fast_start_profiles = self.unit_inputs.get_fast_start_profiles_for_dispatch(dispatch)
        self.market.set_fast_start_constraints(fast_start_profiles)
        if 'fast_start' in self.market._constraints_rhs_and_type:
            cost = self.constraint_inputs.get_constraint_violation_prices()['fast_start']
            self.market.make_constraints_elastic('fast_start', cost)

    def set_unit_fcas_constraints(self):
        self.unit_inputs.add_fcas_trapezium_constraints()

        cost = self.constraint_inputs.get_constraint_violation_prices()['fcas_max_avail']
        fcas_availability = self.unit_inputs.get_fcas_max_availability()
        self.market.set_fcas_max_availability(fcas_availability)
        self.market.make_constraints_elastic('fcas_max_availability', cost)

        cost = self.constraint_inputs.get_constraint_violation_prices()['fcas_profile']

        regulation_trapeziums = self.unit_inputs.get_fcas_regulation_trapeziums()
        self.market.set_energy_and_regulation_capacity_constraints(regulation_trapeziums)
        self.market.make_constraints_elastic('energy_and_regulation_capacity', cost)

        scada_ramp_down_rates = self.unit_inputs.get_scada_ramp_down_rates_of_lower_reg_units()
        self.market.set_joint_ramping_constraints_lower_reg(scada_ramp_down_rates)
        self.market.make_constraints_elastic('joint_ramping_lower_reg', cost)

        scada_ramp_up_rates = self.unit_inputs.get_scada_ramp_up_rates_of_raise_reg_units()
        self.market.set_joint_ramping_constraints_raise_reg(scada_ramp_up_rates)
        self.market.make_constraints_elastic('joint_ramping_raise_reg', cost)

        contingency_trapeziums = self.unit_inputs.get_contingency_services()
        self.market.set_joint_capacity_constraints(contingency_trapeziums)
        self.market.make_constraints_elastic('joint_capacity', cost)

    def set_region_demand_constraints(self):
        regional_demand = self.regional_demand_inputs.get_operational_demand()
        self.market.set_demand_constraints(regional_demand)
        cost = self.constraint_inputs.get_constraint_violation_prices()['regional_demand']
        self.market.make_constraints_elastic('demand', cost)

    def add_interconnectors_to_market(self):
        interconnectors = self.interconnector_inputs.get_interconnector_definitions()
        loss_functions, interpolation_break_points = self.interconnector_inputs.get_interconnector_loss_model()
        self.market.set_interconnectors(interconnectors)
        self.market.set_interconnector_losses(loss_functions, interpolation_break_points)

    def add_generic_constraints_with_fcas_requirements_interface(self):
        fcas_requirements = self.constraint_inputs.get_fcas_requirements()
        self.market.set_fcas_requirements_constraints(fcas_requirements)
        violation_costs = self.constraint_inputs.get_violation_costs()
        self.market.make_constraints_elastic('fcas', violation_cost=violation_costs)
        generic_rhs = self.constraint_inputs.get_rhs_and_type_excluding_regional_fcas_constraints()
        self.market.set_generic_constraints(generic_rhs)
        self.market.make_constraints_elastic('generic', violation_cost=violation_costs)
        unit_generic_lhs = self.constraint_inputs.get_unit_lhs()
        self.market.link_units_to_generic_constraints(unit_generic_lhs)
        interconnector_generic_lhs = self.constraint_inputs.get_interconnector_lhs()
        self.market.link_interconnectors_to_generic_constraints(interconnector_generic_lhs)

    def add_generic_constraints(self):
        violation_costs = self.constraint_inputs.get_violation_costs()
        generic_rhs = self.constraint_inputs.get_rhs_and_type()
        self.market.set_generic_constraints(generic_rhs)
        self.market.make_constraints_elastic('generic', violation_cost=violation_costs)
        unit_generic_lhs = self.constraint_inputs.get_unit_lhs()
        self.market.link_units_to_generic_constraints(unit_generic_lhs)
        interconnector_generic_lhs = self.constraint_inputs.get_interconnector_lhs()
        self.market.link_interconnectors_to_generic_constraints(interconnector_generic_lhs)
        regions_generic_lhs = self.constraint_inputs.get_region_lhs()
        self.market.link_regions_to_generic_constraints(regions_generic_lhs)

    def dispatch(self, calc_prices=True):
        if self.constraint_inputs.is_over_constrained_dispatch_rerun():
            self.market.dispatch(allow_over_constrained_dispatch_re_run=True,
                                 energy_market_floor_price=-1000.0, energy_market_ceiling_price=14500.0,
                                 fcas_market_ceiling_price=1000.0)
        else:
            self.market.dispatch(allow_over_constrained_dispatch_re_run=False)

    def get_market_object(self):
        return self.market


class MarketOverrider:
    def __init__(self, market, mms_db, interval):
        self.services = ['TOTALCLEARED', 'LOWER5MIN', 'LOWER60SEC', 'LOWER6SEC', 'RAISE5MIN', 'RAISE60SEC', 'RAISE6SEC',
                         'LOWERREG', 'RAISEREG']

        self.service_name_mapping = {'TOTALCLEARED': 'energy', 'RAISEREG': 'raise_reg', 'LOWERREG': 'lower_reg',
                                     'RAISE6SEC': 'raise_6s', 'RAISE60SEC': 'raise_60s', 'RAISE5MIN': 'raise_5min',
                                     'LOWER6SEC': 'lower_6s', 'LOWER60SEC': 'lower_60s', 'LOWER5MIN': 'lower_5min',
                                     'ENERGY': 'energy'}

        self.inputs_manager = mms_db
        self.interval = interval

        self.market = market

    def set_unit_dispatch_to_historical_values(self, wiggle_room=0.001):
        DISPATCHLOAD = self.inputs_manager.DISPATCHLOAD.get_data(self.interval)

        bounds = DISPATCHLOAD.loc[:, ['DUID'] + self.services]
        bounds.columns = ['unit'] + self.services

        bounds = hf.stack_columns(bounds, cols_to_keep=['unit'], cols_to_stack=self.services, type_name='service',
                                  value_name='dispatched')

        bounds['service'] = bounds['service'].apply(lambda x: self.service_name_mapping[x])

        decision_variables = self.market._decision_variables['bids'].copy()

        decision_variables = pd.merge(decision_variables, bounds, on=['unit', 'service'])

        decision_variables_first_bid = decision_variables.groupby(['unit', 'service'], as_index=False).first()

        def last_bids(df):
            return df.iloc[1:]

        decision_variables_remaining_bids = \
            decision_variables.groupby(['unit', 'service'], as_index=False).apply(last_bids)

        decision_variables_first_bid['lower_bound'] = decision_variables_first_bid['dispatched'] - wiggle_room
        decision_variables_first_bid['upper_bound'] = decision_variables_first_bid['dispatched'] + wiggle_room
        decision_variables_first_bid['lower_bound'] = np.where(decision_variables_first_bid['lower_bound'] < 0.0, 0.0,
                                                               decision_variables_first_bid['lower_bound'])
        decision_variables_first_bid['upper_bound'] = np.where(decision_variables_first_bid['upper_bound'] < 0.0, 0.0,
                                                               decision_variables_first_bid['upper_bound'])
        decision_variables_remaining_bids['lower_bound'] = 0.0
        decision_variables_remaining_bids['upper_bound'] = 0.0

        decision_variables = pd.concat([decision_variables_first_bid, decision_variables_remaining_bids])

        self.market._decision_variables['bids'] = decision_variables

    def set_interconnector_flow_to_historical_values(self, wiggle_room=0.1):
        # Historical interconnector dispatch
        DISPATCHINTERCONNECTORRES = self.inputs_manager.DISPATCHINTERCONNECTORRES.get_data(self.interval)
        interconnector_flow = DISPATCHINTERCONNECTORRES.loc[:, ['INTERCONNECTORID', 'MWFLOW']]
        interconnector_flow.columns = ['interconnector', 'flow']
        interconnector_flow['link'] = interconnector_flow['interconnector']
        interconnector_flow['link'] = np.where(interconnector_flow['interconnector'] == 'T-V-MNSP1',
                                               np.where(interconnector_flow['flow'] >= 0.0, 'BLNKTAS', 'BLNKVIC'),
                                               interconnector_flow['link'])

        flow_variables = self.market._decision_variables['interconnectors']
        flow_variables = pd.merge(flow_variables, interconnector_flow, 'left', on=['interconnector', 'link'])
        flow_variables = flow_variables.fillna(0.0)
        flow_variables['flow'] = np.where(flow_variables['link'] != flow_variables['interconnector'],
                                          flow_variables['flow'].abs(), flow_variables['flow'])
        flow_variables['lower_bound'] = flow_variables['flow'] - wiggle_room
        flow_variables['upper_bound'] = flow_variables['flow'] + wiggle_room
        flow_variables = flow_variables.drop(['flow'], axis=1)
        self.market._decision_variables['interconnectors'] = flow_variables


class MarketChecker:
    def __init__(self, market, mms_db, xml_cache, interval, unit_inputs=None):
        self.services = ['TOTALCLEARED', 'LOWER5MIN', 'LOWER60SEC', 'LOWER6SEC', 'RAISE5MIN', 'RAISE60SEC', 'RAISE6SEC',
                         'LOWERREG', 'RAISEREG']

        self.service_name_mapping = {'TOTALCLEARED': 'energy', 'RAISEREG': 'raise_reg', 'LOWERREG': 'lower_reg',
                                     'RAISE6SEC': 'raise_6s', 'RAISE60SEC': 'raise_60s', 'RAISE5MIN': 'raise_5min',
                                     'LOWER6SEC': 'lower_6s', 'LOWER60SEC': 'lower_60s', 'LOWER5MIN': 'lower_5min',
                                     'ENERGY': 'energy'}

        self.inputs_manager = mms_db
        self.xml = xml_cache
        self.unit_inputs = unit_inputs
        self.interval = interval

        self.market = market

    def all_dispatch_units_and_services_have_decision_variables(self, wiggle_room=0.001):
        DISPATCHLOAD = self.inputs_manager.DISPATCHLOAD.get_data(self.interval)

        bounds = DISPATCHLOAD.loc[:, ['DUID'] + self.services]
        bounds.columns = ['unit'] + self.services

        bounds = hf.stack_columns(bounds, cols_to_keep=['unit'], cols_to_stack=self.services, type_name='service',
                                  value_name='dispatched')

        bounds['service'] = bounds['service'].apply(lambda x: self.service_name_mapping[x])

        bounds = bounds[bounds['dispatched'] > 0.001]

        decision_variables = self.market._decision_variables['bids'].copy()

        decision_variables = decision_variables.groupby(['unit', 'service'], as_index=False).first()

        decision_variables = pd.merge(bounds, decision_variables, how='left', on=['unit', 'service'])

        decision_variables['not_missing'] = ~decision_variables['variable_id'].isna()

        decision_variables = decision_variables.sort_values('not_missing')

        return decision_variables['not_missing'].all()

    def is_regional_demand_meet(self, tolerance=0.5):
        DISPATCHREGIONSUM = self.inputs_manager.DISPATCHREGIONSUM.get_data(self.interval)
        regional_demand = demand._format_regional_demand(DISPATCHREGIONSUM)
        region_summary = self.market.get_region_dispatch_summary()
        region_summary = pd.merge(region_summary, regional_demand, on='region')
        region_summary['calc_demand'] = region_summary['dispatch'] + region_summary['inflow'] \
            - region_summary['interconnector_losses'] - region_summary['transmission_losses']
        region_summary['diff'] = region_summary['calc_demand'] - region_summary['demand']
        region_summary['no_error'] = region_summary['diff'].abs() < tolerance
        return region_summary['no_error'].all()

    def is_generic_constraint_slack_correct(self):

        def calc_slack(rhs, lhs, type):
            if type == '<=':
                slack = rhs - lhs
            elif type == '>=':
                slack = lhs - rhs
            else:
                slack = 0.0
            if slack < 0.0:
                slack = 0.0
            return slack

        DISPATCHCONSTRAINT = self.inputs_manager.DISPATCHCONSTRAINT.get_data(self.interval)
        generic_cons_slack = self.market._constraints_rhs_and_type['generic']
        generic_cons_slack['slack'] = generic_cons_slack['slack'].abs()
        generic_cons_slack = pd.merge(generic_cons_slack, DISPATCHCONSTRAINT, left_on='set',
                                      right_on='CONSTRAINTID')
        generic_cons_slack['aemo_slack'] = (generic_cons_slack['RHS'] - generic_cons_slack['LHS'])
        generic_cons_slack['aemo_slack'] = \
            generic_cons_slack.apply(lambda x: calc_slack(x['RHS'], x['LHS'], x['type']), axis=1)
        generic_cons_slack['comp'] = (generic_cons_slack['aemo_slack'] - generic_cons_slack['slack']).abs()
        generic_cons_slack['no_error'] = generic_cons_slack['comp'] < 0.9
        return generic_cons_slack['no_error'].all()

    def is_fcas_constraint_slack_correct(self):

        def calc_slack(rhs, lhs, type):
            if type == '<=':
                slack = rhs - lhs
            elif type == '>=':
                slack = lhs - rhs
            else:
                slack = 0.0
            if slack < 0.0:
                slack = 0.0
            return slack

        DISPATCHCONSTRAINT = self.inputs_manager.DISPATCHCONSTRAINT.get_data(self.interval)
        generic_cons_slack = self.market._market_constraints_rhs_and_type['fcas']
        generic_cons_slack['slack'] = generic_cons_slack['slack'].abs()
        generic_cons_slack = pd.merge(generic_cons_slack, DISPATCHCONSTRAINT, left_on='set',
                                      right_on='CONSTRAINTID')
        generic_cons_slack['aemo_slack'] = (generic_cons_slack['RHS'] - generic_cons_slack['LHS'])
        generic_cons_slack['aemo_slack'] = \
            generic_cons_slack.apply(lambda x: calc_slack(x['RHS'], x['LHS'], x['type']), axis=1)
        generic_cons_slack['comp'] = (generic_cons_slack['aemo_slack'] - generic_cons_slack['slack']).abs()
        generic_cons_slack['no_error'] = generic_cons_slack['comp'] < 0.9
        return generic_cons_slack['no_error'].all()

    def all_constraints_presenet(self):
        DISPATCHCONSTRAINT = list(self.inputs_manager.DISPATCHCONSTRAINT.get_data(self.interval)['CONSTRAINTID'])
        fcas = list(self.market._market_constraints_rhs_and_type['fcas']['set'])
        generic = list(self.market._constraints_rhs_and_type['generic']['set'])
        generic = generic + fcas
        return set(DISPATCHCONSTRAINT) < set(generic + [1])

    def get_price_comparison(self):
        energy_prices = self.market.get_energy_prices()
        energy_prices['time'] = self.interval
        energy_prices['service'] = 'energy'
        fcas_prices = self.market.get_fcas_prices()
        fcas_prices['time'] = self.interval
        prices = pd.concat([energy_prices, fcas_prices])

        price_to_service = {'ROP': 'energy', 'RAISE6SECROP': 'raise_6s', 'RAISE60SECROP': 'raise_60s',
                            'RAISE5MINROP': 'raise_5min', 'RAISEREGROP': 'raise_reg', 'LOWER6SECROP': 'lower_6s',
                            'LOWER60SECROP': 'lower_60s', 'LOWER5MINROP': 'lower_5min', 'LOWERREGROP': 'lower_reg'}
        price_columns = list(price_to_service.keys())
        historical_prices = self.inputs_manager.DISPATCHPRICE.get_data(self.interval)
        historical_prices = hf.stack_columns(historical_prices, cols_to_keep=['SETTLEMENTDATE', 'REGIONID'],
                                             cols_to_stack=price_columns, type_name='service',
                                             value_name='RRP')
        historical_prices['service'] = historical_prices['service'].apply(lambda x: price_to_service[x])
        historical_prices = historical_prices.loc[:, ['SETTLEMENTDATE', 'REGIONID', 'service', 'RRP']]
        historical_prices.columns = ['time', 'region', 'service', 'hist_price']
        prices = pd.merge(prices, historical_prices, on=['time', 'region', 'service'])
        prices['error'] = prices['price'] - prices['hist_price']
        return prices

    def get_dispatch_comparison(self):
        DISPATCHLOAD = self.inputs_manager.DISPATCHLOAD.get_data(self.interval)
        bounds = DISPATCHLOAD.loc[:, ['DUID'] + self.services]
        bounds.columns = ['unit'] + self.services
        bounds = hf.stack_columns(bounds, cols_to_keep=['unit'], cols_to_stack=self.services, type_name='service',
                                  value_name='dispatched')
        bounds['service'] = bounds['service'].apply(lambda x: self.service_name_mapping[x])

        nempy_dispatch = self.market.get_unit_dispatch()
        comp = pd.merge(bounds, nempy_dispatch, 'inner', on=['unit', 'service'])
        comp['diff'] = comp['dispatch'] - comp['dispatched']
        comp = pd.merge(comp, self.market._unit_info.loc[:, ['unit', 'dispatch_type']], on='unit')
        comp['diff'] = np.where((comp['dispatch_type'] == 'load') & (comp['service'] == 'energy'), comp['diff'] * -1,
                                comp['diff'])
        return comp

    def do_fcas_availabilities_match_historical(self):
        DISPATCHLOAD = self.inputs_manager.DISPATCHLOAD.get_data(self.interval)
        availabilities = ['RAISE6SECACTUALAVAILABILITY', 'RAISE60SECACTUALAVAILABILITY',
                          'RAISE5MINACTUALAVAILABILITY', 'RAISEREGACTUALAVAILABILITY',
                          'LOWER6SECACTUALAVAILABILITY', 'LOWER60SECACTUALAVAILABILITY',
                          'LOWER5MINACTUALAVAILABILITY', 'LOWERREGACTUALAVAILABILITY']

        availabilities_mapping = {'RAISEREGACTUALAVAILABILITY': 'raise_reg',
                                  'LOWERREGACTUALAVAILABILITY': 'lower_reg',
                                  'RAISE6SECACTUALAVAILABILITY': 'raise_6s',
                                  'RAISE60SECACTUALAVAILABILITY': 'raise_60s',
                                  'RAISE5MINACTUALAVAILABILITY': 'raise_5min',
                                  'LOWER6SECACTUALAVAILABILITY': 'lower_6s',
                                  'LOWER60SECACTUALAVAILABILITY': 'lower_60s',
                                  'LOWER5MINACTUALAVAILABILITY': 'lower_5min'}

        bounds = DISPATCHLOAD.loc[:, ['DUID'] + availabilities]
        bounds.columns = ['unit'] + availabilities

        availabilities = hf.stack_columns(bounds, cols_to_keep=['unit'], cols_to_stack=availabilities,
                                          type_name='service', value_name='availability')

        bounds = DISPATCHLOAD.loc[:, ['DUID'] + self.services]
        bounds.columns = ['unit'] + self.services

        bounds = hf.stack_columns(bounds, cols_to_keep=['unit'], cols_to_stack=self.services, type_name='service',
                                  value_name='dispatched')

        bounds['service'] = bounds['service'].apply(lambda x: self.service_name_mapping[x])

        availabilities['service'] = availabilities['service'].apply(lambda x: availabilities_mapping[x])

        #availabilities = pd.merge(availabilities, bounds, on=['unit', 'service'])

        #availabilities = availabilities[~(availabilities['dispatched'] - 0.001 > availabilities['availability'])]

        output = self.market.get_fcas_availability()
        output.columns = ['unit', 'service', 'availability_measured']

        availabilities = pd.merge(availabilities, output, 'left', on=['unit', 'service'])

        availabilities['availability_measured'] = availabilities['availability_measured'].fillna(0)

        availabilities['error'] = availabilities['availability_measured'] - availabilities['availability']

        availabilities['match'] = availabilities['error'].abs() < 0.1
        availabilities = availabilities.sort_values('match')

        return availabilities

    def measured_violation_equals_historical_violation(self, historical_name, nempy_constraints):
        measured = 0.0
        for name in nempy_constraints:
            measured += self.market.get_elastic_constraints_violation_degree(name)
        historical = self.xml.get_violations()[historical_name]
        return measured == pytest.approx(historical, abs=0.1)

================
File: tests\test_constraint_equation_calc.py
================
import pytest
from datetime import datetime, timedelta
import random
import pandas as pd
import os

from nempy.historical_inputs.rhs_calculator import RHSCalc
from nempy.historical_inputs import xml_cache


def test_single_equation():
    xml_cache_manager = xml_cache.XMLCacheManager('nemde_cache_rhs_calc_testing')
    xml_cache_manager.load_interval('2013/01/04 12:40:00')
    rhs_calculator = RHSCalc(xml_cache_manager)
    print(xml_cache_manager.get_file_path())
    con = 'DATASNAP'
    print(rhs_calculator.compute_constraint_rhs(con) - rhs_calculator.get_nemde_rhs(con))
    assert (rhs_calculator.compute_constraint_rhs(con) ==
            pytest.approx(rhs_calculator.get_nemde_rhs(con), 0.001))


def test_rhs_equations_in_order_of_length_all():
    xml_cache_manager = xml_cache.XMLCacheManager('nemde_cache_rhs_calc_testing')

    def get_test_intervals(number=100):
        start_time = datetime(year=2013, month=1, day=1, hour=0, minute=0)
        end_time = datetime(year=2013, month=1, day=5, hour=0, minute=0)
        difference = end_time - start_time
        difference_in_5_min_intervals = difference.days * 12 * 24
        random.seed(1)
        intervals = random.sample(range(1, difference_in_5_min_intervals), number)
        times = [start_time + timedelta(minutes=5 * i) for i in intervals]
        times_formatted = [t.isoformat().replace('T', ' ').replace('-', '/') for t in times]
        return times_formatted

    intervals = []
    rhs_ids = []
    nemde_rhs = []
    nempy_rhs = []
    files = []
    basslink_dep_equations = set()

    test_context = 'online'  # set to local to run more extensive tests

    if test_context == 'online':
        intervals_to_test = ['2013/01/01 00:00:00']
    else:
        intervals_to_test = get_test_intervals(100)

    for interval in intervals_to_test:
        xml_cache_manager.load_interval(interval)
        rhs_calculator = RHSCalc(xml_cache_manager)
        rhs_equations = rhs_calculator.rhs_constraint_equations
        equations_in_length_order = sorted(rhs_equations, key=lambda k: len(rhs_equations[k]))
        xml_file = xml_cache_manager.get_file_path()
        for equation_id in equations_in_length_order:
            basslink_dep_equations = basslink_dep_equations.union(set(
                rhs_calculator.get_rhs_constraint_equations_that_depend_value('BL_FREQ_ONSTATUS', 'W')))
            try:
                result = rhs_calculator.compute_constraint_rhs(equation_id)
                if not (rhs_calculator.get_nemde_rhs(equation_id) == pytest.approx(result, 0.001)):
                    intervals.append(interval)
                    rhs_ids.append(equation_id)
                    nemde_rhs.append(rhs_calculator.get_nemde_rhs(equation_id))
                    nempy_rhs.append(rhs_calculator.compute_constraint_rhs(equation_id))
                    files.append(xml_file)
            except:
                result = 'error'
                intervals.append(interval)
                rhs_ids.append(equation_id)
                nemde_rhs.append(rhs_calculator.get_nemde_rhs(equation_id))
                nempy_rhs.append(result)
                files.append(xml_file)

    errors = pd.DataFrame({
        'intervals': intervals,
        'rhs_id': rhs_ids,
        'nemde_rhs': nemde_rhs,
        'nempy_rhs': nempy_rhs,
        'files': files
    })

    error_raises = errors[errors['nempy_rhs'] == 'error'].copy()
    assert len(error_raises['rhs_id'].unique()) <= 4
    error_raises.to_csv('rhs_calculation_error_raises.csv')

    errors = errors[errors['nempy_rhs'] != 'error'].copy()

    errors['error'] = (errors['nemde_rhs'] - errors['nempy_rhs']).abs()

    errors.to_csv('rhs_calculation_errors.csv')

    basslink_errors = errors[errors['rhs_id'].isin(basslink_dep_equations)]
    basslink_errors.to_csv('basslink_rhs_calculation_errors.csv')

    errors_summary = errors.groupby('rhs_id', as_index=False).agg({'error': 'max'})
    errors_summary.to_csv('rhs_calculation_errors_summary.csv')

    basslink_dep_equations = list(basslink_dep_equations)

    basslink_errors_summary = errors_summary[errors_summary['rhs_id'].isin(basslink_dep_equations)]
    assert len(basslink_errors_summary) <= 5
    basslink_errors_summary.to_csv('rhs_calculation_basslink_errors_summary.csv')

    basslink_error_raises = error_raises[error_raises['rhs_id'].isin(basslink_dep_equations)]
    assert len(basslink_error_raises) == 0
    basslink_error_raises.to_csv('basslink_error_raises.csv')


def test_get_constraints_that_depend_on_500_GEN_INERTIA():
    xml_cache_manager = xml_cache.XMLCacheManager('nemde_cache_rhs_calc_testing')
    xml_cache_manager.load_interval('2013/01/01 00:00:00')
    rhs_calculator = RHSCalc(xml_cache_manager)
    cons_that_depend_on_value = (
        rhs_calculator.get_rhs_constraint_equations_that_depend_value('500_GEN_INERTIA', 'A'))
    expected_cons = ["V::N_NIL_Q1", "V::N_NIL_Q2", "V::N_NIL_Q3", "V::N_NIL_Q4", "V::N_NIL_S1", "V::N_NIL_S2",
                     "V::N_NIL_S3", "V::N_NIL_S4", "V::N_NIL_V1", "V::N_NIL_V2", "V::N_NIL_V3", "V::N_NIL_V4"]
    assert expected_cons == cons_that_depend_on_value


def test_get_constraints_that_depend_on_BL_FREQ_ONSTATUS():
    xml_cache_manager = xml_cache.XMLCacheManager('nemde_cache_rhs_calc_testing')
    xml_cache_manager.load_interval('2013/01/01 00:00:00')
    rhs_calculator = RHSCalc(xml_cache_manager)
    cons_that_depend_on_value = (
        rhs_calculator.get_rhs_constraint_equations_that_depend_value('BL_FREQ_ONSTATUS', 'W'))
    # Intermediate generic equations
    # ["X_BL_MAIN_MAXAVL_OFF", "X_BL_MAIN_MAXAVL_ON", "X_BL_MAIN_MINAVL_OFF", "X_BL_MAIN_MINAVL_ON",
    #  "X_BL_TAS_MAXAVL_OFF", "X_BL_TAS_MAXAVL_ON", "X_BL_TAS_MINAVL_OFF", "X_BL_TAS_MINAVL_ON"]
    expected_cons = ["F_MAIN+NIL_DYN_RREG", "F_MAIN+NIL_MG_R5", "F_MAIN+NIL_MG_R6", "F_MAIN+NIL_MG_R60",
     "F_MAIN++NIL_DYN_RREG", "F_MAIN++NIL_MG_R5", "F_MAIN++NIL_MG_R6", "F_MAIN++NIL_MG_R60",
     "F_MAIN+ML_L5_0400", "F_MAIN+ML_L60_0400", "F_MAIN+ML_L6_0400", "F_MAIN+NIL_DYN_LREG",
     "F_MAIN++ML_L5_0400", "F_MAIN++ML_L60_0400", "F_MAIN++ML_L6_0400", "F_MAIN++NIL_DYN_LREG",
     "F_T+LREG_0050", "F_T+NIL_ML_L5", "F_T+NIL_ML_L6", "F_T+NIL_ML_L60", "F_T+NIL_TL_L5", "F_T+NIL_TL_L6",
     "F_T+NIL_TL_L60", "F_T++LREG_0050", "F_T++NIL_ML_L5", "F_T++NIL_ML_L6", "F_T++NIL_ML_L60", "F_T++NIL_TL_L5",
     "F_T++NIL_TL_L6", "F_T++NIL_TL_L60", "F_T+NIL_BB_TG_R5", "F_T+NIL_BB_TG_R6", "F_T+NIL_BB_TG_R60",
     "F_T+NIL_MG_R5", "F_T+NIL_MG_R6", "F_T+NIL_MG_R60", "F_T+NIL_WN_TG_R5", "F_T+NIL_WN_TG_R6", "F_T+NIL_WN_TG_R60",
     "F_T+RREG_0050",
     "F_T++NIL_BB_TG_R5", "F_T++NIL_BB_TG_R6", "F_T++NIL_BB_TG_R60", "F_T++NIL_MG_R5", "F_T++NIL_MG_R6",
     "F_T++NIL_MG_R60", "F_T++NIL_WN_TG_R5", "F_T++NIL_WN_TG_R6", "F_T++NIL_WN_TG_R60", "F_T++RREG_0050",
     "T_V_NIL_BL1", "V_T_NIL_BL1"]
    assert sorted(expected_cons) == sorted(cons_that_depend_on_value)

================
File: tests\test_historical.py
================
import sqlite3
import pandas as pd
from pandas.testing import assert_frame_equal
from datetime import datetime, timedelta
import random
import pickle
from nempy.historical_inputs import loaders, xml_cache, mms_db, units, \
    interconnectors, constraints, demand
from tests import historical_market_builder

test_db = 'D:/nempy_test_files/historical_mms.db'
test_xml_cache = 'D:/nempy_test_files/nemde_cache'


# These tests require some additional clean up and will probably not run on your machine. ##############################


def get_test_intervals(number=100):
    start_time = datetime(year=2019, month=1, day=1, hour=0, minute=0)
    end_time = datetime(year=2019, month=12, day=31, hour=0, minute=0)
    difference = end_time - start_time
    difference_in_5_min_intervals = difference.days * 12 * 24
    random.seed(2)
    intervals = random.sample(range(1, difference_in_5_min_intervals), number)
    times = [start_time + timedelta(minutes=5 * i) for i in intervals]
    times_formatted = [t.isoformat().replace('T', ' ').replace('-', '/') for t in times]
    return times_formatted


def get_test_intervals_august_2020(number=100):
    start_time = datetime(year=2020, month=8, day=1, hour=0, minute=0)
    end_time = datetime(year=2020, month=8, day=31, hour=0, minute=0)
    difference = end_time - start_time
    difference_in_5_min_intervals = difference.days * 12 * 24
    random.seed(2)
    intervals = random.sample(range(1, difference_in_5_min_intervals), number)
    times = [start_time + timedelta(minutes=5 * i) for i in intervals]
    times_formatted = [t.isoformat().replace('T', ' ').replace('-', '/') for t in times]
    return times_formatted


def test_xml_price_bid_loader_vs_mms_db_loader():
    test_db = 'F:/nempy_test_files/historical_mms.db'
    test_xml_cache = 'F:/nempy_test_files/nemde_cache'
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    xml_cache_manager.load_interval('2019/01/01 00:00:00')
    xml_bids = xml_cache_manager.get_unit_price_bids()
    mms_bids = mms_database.BIDDAYOFFER_D.get_data('2019/01/01 00:00:00')
    mms_bids = mms_bids.loc[:, ['DUID', 'BIDTYPE', 'PRICEBAND1', 'PRICEBAND2', 'PRICEBAND3', 'PRICEBAND4', 'PRICEBAND5',
                                'PRICEBAND6', 'PRICEBAND7', 'PRICEBAND8', 'PRICEBAND9',  'PRICEBAND10']]
    mms_bids = mms_bids.sort_values(['DUID', 'BIDTYPE'])
    xml_bids = xml_bids.sort_values(['DUID', 'BIDTYPE'])
    assert_frame_equal(xml_bids.reset_index(drop=True), mms_bids.reset_index(drop=True), check_less_precise=3)


def test_ramp_rate_constraints():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)

    for interval in get_test_intervals(number=10):
        raw_inputs_loader.set_interval(interval)
        unit_inputs = units.UnitData(raw_inputs_loader)
        interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
        constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
        demand_inputs = demand.DemandData(raw_inputs_loader)

        market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                     interconnector_inputs=interconnector_inputs,
                                                                     constraint_inputs=constraint_inputs,
                                                                     demand_inputs=demand_inputs)
        market_builder.add_unit_bids_to_market()
        market_builder.set_ramp_rate_limits()

        market = market_builder.get_market_object()

        market_overrider = historical_market_builder.MarketOverrider(market=market,
                                                                     mms_db=mms_database,
                                                                     interval=interval)

        market_overrider.set_unit_dispatch_to_historical_values()

        market_builder.dispatch()

        market_checker = historical_market_builder.MarketChecker(market=market,
                                                                 mms_db=mms_database,
                                                                 xml_cache=xml_cache_manager,
                                                                 interval=interval)

        assert market_checker.measured_violation_equals_historical_violation(historical_name='ramp_rate',
                                                                             nempy_constraints=['ramp_up', 'ramp_down'])


def test_ramp_rate_constraints_where_constraints_violated():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)

    with open('interval_with_violations.pickle', 'rb') as f:
        interval_with_violations = pickle.load(f)

    tests_to_run = 55
    tests_run = 0
    for interval, types in interval_with_violations.items():
        if tests_run == tests_to_run:
            break
        if 'ramp_rate' in types:
            raw_inputs_loader.set_interval(interval)
            unit_inputs = units.UnitData(raw_inputs_loader)
            interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
            constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
            demand_inputs = demand.DemandData(raw_inputs_loader)

            market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                         interconnector_inputs=interconnector_inputs,
                                                                         constraint_inputs=constraint_inputs,
                                                                         demand_inputs=demand_inputs)
            market_builder.add_unit_bids_to_market()
            market_builder.set_ramp_rate_limits()

            market = market_builder.get_market_object()

            market_overrider = historical_market_builder.MarketOverrider(market=market,
                                                                         mms_db=mms_database,
                                                                         interval=interval)

            market_overrider.set_unit_dispatch_to_historical_values()

            market_builder.dispatch()

            market_checker = historical_market_builder.MarketChecker(market=market,
                                                                     mms_db=mms_database,
                                                                     xml_cache=xml_cache_manager,
                                                                     interval=interval)

            assert market_checker.measured_violation_equals_historical_violation(historical_name='ramp_rate',
                                                                                 nempy_constraints=['ramp_up',
                                                                                                    'ramp_down'])
            tests_run += 1

    assert tests_to_run == tests_run


def test_fast_start_constraints():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)

    for interval in get_test_intervals(number=10):
        raw_inputs_loader.set_interval(interval)
        unit_inputs = units.UnitData(raw_inputs_loader)
        interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
        constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
        demand_inputs = demand.DemandData(raw_inputs_loader)

        market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                     interconnector_inputs=interconnector_inputs,
                                                                     constraint_inputs=constraint_inputs,
                                                                     demand_inputs=demand_inputs)
        market_builder.add_unit_bids_to_market()
        market_builder.set_fast_start_constraints()

        market = market_builder.get_market_object()

        market_overrider = historical_market_builder.MarketOverrider(market=market,
                                                                     mms_db=mms_database,
                                                                     interval=interval)

        market_overrider.set_unit_dispatch_to_historical_values()

        market_builder.dispatch()

        market_checker = historical_market_builder.MarketChecker(market=market,
                                                                 mms_db=mms_database,
                                                                 xml_cache=xml_cache_manager,
                                                                 interval=interval)

        assert market_checker.measured_violation_equals_historical_violation('fast_start',
                                                                             nempy_constraints=['fast_start'])


def test_fast_start_constraints_where_constraints_violated():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)

    with open('interval_with_violations.pickle', 'rb') as f:
        interval_with_violations = pickle.load(f)

    tests_to_run = 11
    tests_run = 0
    for interval, types in interval_with_violations.items():
        if tests_run == tests_to_run:
            break
        if 'fast_start' in types:
            raw_inputs_loader.set_interval(interval)
            unit_inputs = units.UnitData(raw_inputs_loader)
            interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
            constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
            demand_inputs = demand.DemandData(raw_inputs_loader)

            market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                         interconnector_inputs=interconnector_inputs,
                                                                         constraint_inputs=constraint_inputs,
                                                                         demand_inputs=demand_inputs)
            market_builder.add_unit_bids_to_market()
            market_builder.set_fast_start_constraints()

            market = market_builder.get_market_object()

            market_overrider = historical_market_builder.MarketOverrider(market=market,
                                                                         mms_db=mms_database,
                                                                         interval=interval)

            market_overrider.set_unit_dispatch_to_historical_values()

            market_builder.dispatch()

            market_checker = historical_market_builder.MarketChecker(market=market,
                                                                     mms_db=mms_database,
                                                                     xml_cache=xml_cache_manager,
                                                                     interval=interval)

            assert market_checker.measured_violation_equals_historical_violation('fast_start',
                                                                                 nempy_constraints=[
                                                                                     'fast_start'])
            tests_run += 1

    assert tests_to_run == tests_run


def test_capacity_constraints():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)

    for interval in get_test_intervals(number=10):
        raw_inputs_loader.set_interval(interval)
        unit_inputs = units.UnitData(raw_inputs_loader)
        interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
        constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
        demand_inputs = demand.DemandData(raw_inputs_loader)

        market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                     interconnector_inputs=interconnector_inputs,
                                                                     constraint_inputs=constraint_inputs,
                                                                     demand_inputs=demand_inputs)
        market_builder.add_unit_bids_to_market()
        market_builder.add_interconnectors_to_market()
        market_builder.set_unit_limit_constraints()

        market = market_builder.get_market_object()

        market_overrider = historical_market_builder.MarketOverrider(market=market,
                                                                     mms_db=mms_database,
                                                                     interval=interval)

        market_overrider.set_unit_dispatch_to_historical_values()
        market_overrider.set_interconnector_flow_to_historical_values()

        market_builder.dispatch()

        market_checker = historical_market_builder.MarketChecker(market=market,
                                                                 mms_db=mms_database,
                                                                 xml_cache=xml_cache_manager,
                                                                 interval=interval)

        assert market_checker.measured_violation_equals_historical_violation('unit_capacity',
                                                                             nempy_constraints=['unit_bid_capacity'])


def test_capacity_constraint_where_constraints_violated():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)

    with open('interval_with_violations.pickle', 'rb') as f:
        interval_with_violations = pickle.load(f)

    tests_to_run = 10
    tests_run = 0
    for interval, types in interval_with_violations.items():
        if tests_run == tests_to_run:
            break
        if 'unit_capacity' in types:
            raw_inputs_loader.set_interval(interval)
            unit_inputs = units.UnitData(raw_inputs_loader)
            interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
            constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
            demand_inputs = demand.DemandData(raw_inputs_loader)

            market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                         interconnector_inputs=interconnector_inputs,
                                                                         constraint_inputs=constraint_inputs,
                                                                         demand_inputs=demand_inputs)
            market_builder.add_unit_bids_to_market()
            market_builder.add_interconnectors_to_market()
            market_builder.set_unit_limit_constraints()

            market = market_builder.get_market_object()

            market_overrider = historical_market_builder.MarketOverrider(market=market,
                                                                         mms_db=mms_database,
                                                                         interval=interval)

            market_overrider.set_unit_dispatch_to_historical_values()
            market_overrider.set_interconnector_flow_to_historical_values()

            market_builder.dispatch()

            market_checker = historical_market_builder.MarketChecker(market=market,
                                                                     mms_db=mms_database,
                                                                     xml_cache=xml_cache_manager,
                                                                     interval=interval)

            assert market_checker.measured_violation_equals_historical_violation('unit_capacity',
                                                                                 nempy_constraints=[
                                                                                     'unit_bid_capacity'])
            tests_run += 1

    assert tests_to_run == tests_run


def ignore_test_fcas_trapezium_scaled_availability():
    con = sqlite3.connect('/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms_august_2020.db')
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager('/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache_august_2020')
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)

    for interval in get_test_intervals_august_2020(number=10):
        if interval != '2020/08/21 13:00:00':
            continue
        raw_inputs_loader.set_interval(interval)
        unit_inputs = units.UnitData(raw_inputs_loader)
        interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
        constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
        demand_inputs = demand.DemandData(raw_inputs_loader)

        market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                     interconnector_inputs=interconnector_inputs,
                                                                     constraint_inputs=constraint_inputs,
                                                                     demand_inputs=demand_inputs)
        market_builder.add_unit_bids_to_market()
        market_builder.set_unit_fcas_constraints()
        market_builder.set_unit_limit_constraints()

        market = market_builder.get_market_object()

        market_overrider = historical_market_builder.MarketOverrider(market=market,
                                                                     mms_db=mms_database,
                                                                     interval=interval)

        market_overrider.set_unit_dispatch_to_historical_values()

        market_builder.dispatch()

        market_checker = historical_market_builder.MarketChecker(market=market,
                                                                 mms_db=mms_database,
                                                                 xml_cache=xml_cache,
                                                                 interval=interval,
                                                                 unit_inputs=unit_inputs)

        avails = market_checker.do_fcas_availabilities_match_historical()
        # I think NEMDE might be getting avail calcs wrong when units are operating on the slopes, and the slopes
        # are vertical. They should be ignore 0 slope coefficients, maybe this is not happening because of floating
        # point comparison.
        if interval == '2019/01/29 18:10:00':
            avails = avails[~(avails['unit'] == 'PPCCGT')]
        if interval == '2019/01/07 19:35:00':
            avails = avails[~(avails['unit'] == 'PPCCGT')]
        #assert avails['error'].abs().max() < 1.1


def ignore_test_find_fcas_trapezium_scaled_availability_erros():
    con = sqlite3.connect('/media/nickgorman/Samsung_T5/nempy_test_files/historical_mms_august_2020.db')
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager('/media/nickgorman/Samsung_T5/nempy_test_files/nemde_cache_august_2020')
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)
    outputs = []
    for interval in get_test_intervals_august_2020(number=100):
        raw_inputs_loader.set_interval(interval)
        unit_inputs = units.UnitData(raw_inputs_loader)
        unit_inputs.get_processed_bids()
        unit_inputs.add_fcas_trapezium_constraints()
        traps = unit_inputs.get_fcas_regulation_trapeziums()
        traps = traps[traps['service'] == 'lower_reg']
        avails = mms_database.DISPATCHLOAD.get_data(interval)
        avails = avails.loc[:, ['DUID', 'TOTALCLEARED', 'LOWERREG', 'LOWERREGACTUALAVAILABILITY']]
        avails.columns = ['unit', 'total_cleared', 'lower_reg', 'lower_reg_actual_availability']
        avails = avails[avails['lower_reg'] > avails['lower_reg_actual_availability'] + 0.1]
        avails = pd.merge(avails, traps, on='unit')
        avails['time'] = interval
        outputs.append(avails)
    pd.concat(outputs).to_csv('avails_august_2020.csv')


def test_all_units_and_service_dispatch_historically_present_in_market():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)

    for interval in get_test_intervals(number=1000):
        raw_inputs_loader.set_interval(interval)
        unit_inputs = units.UnitData(raw_inputs_loader)
        interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
        constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
        demand_inputs = demand.DemandData(raw_inputs_loader)

        market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                     interconnector_inputs=interconnector_inputs,
                                                                     constraint_inputs=constraint_inputs,
                                                                     demand_inputs=demand_inputs)
        market_builder.add_unit_bids_to_market()
        market = market_builder.get_market_object()
        market_checker = historical_market_builder.MarketChecker(market=market,
                                                                 mms_db=mms_database,
                                                                 xml_cache=xml_cache,
                                                                 interval=interval)
        assert market_checker.all_dispatch_units_and_services_have_decision_variables()


def test_slack_in_generic_constraints():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)

    for interval in get_test_intervals(number=100):
        raw_inputs_loader.set_interval(interval)
        unit_inputs = units.UnitData(raw_inputs_loader)
        interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
        constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
        demand_inputs = demand.DemandData(raw_inputs_loader)

        market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                     interconnector_inputs=interconnector_inputs,
                                                                     constraint_inputs=constraint_inputs,
                                                                     demand_inputs=demand_inputs)
        market_builder.add_unit_bids_to_market()
        market_builder.add_interconnectors_to_market()
        market_builder.add_generic_constraints()
        market_builder.set_unit_fcas_constraints()
        market_builder.set_unit_limit_constraints()
        market_builder.set_region_demand_constraints()
        market_builder.set_ramp_rate_limits()
        market_builder.set_fast_start_constraints()
        market_builder.set_solver('CBC')
        market_builder.dispatch(calc_prices=True)
        market = market_builder.get_market_object()

        market_overrider = historical_market_builder.MarketOverrider(market=market,
                                                                     mms_db=mms_database,
                                                                     interval=interval)

        market_overrider.set_unit_dispatch_to_historical_values()
        market_overrider.set_interconnector_flow_to_historical_values()

        market_builder.dispatch()

        market_checker = historical_market_builder.MarketChecker(market=market,
                                                                 mms_db=mms_database,
                                                                 xml_cache=xml_cache,
                                                                 interval=interval)
        assert market_checker.is_generic_constraint_slack_correct()
        assert market_checker.is_regional_demand_meet()


def test_slack_in_generic_constraints_with_fcas_interface():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)

    for interval in get_test_intervals(number=100):
        raw_inputs_loader.set_interval(interval)
        unit_inputs = units.UnitData(raw_inputs_loader)
        interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
        constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
        demand_inputs = demand.DemandData(raw_inputs_loader)

        market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                     interconnector_inputs=interconnector_inputs,
                                                                     constraint_inputs=constraint_inputs,
                                                                     demand_inputs=demand_inputs)
        market_builder.add_unit_bids_to_market()
        market_builder.add_interconnectors_to_market()
        market_builder.add_generic_constraints_with_fcas_requirements_interface()
        market_builder.set_unit_fcas_constraints()
        market_builder.set_unit_limit_constraints()
        market_builder.set_region_demand_constraints()
        market_builder.set_ramp_rate_limits()
        market_builder.set_fast_start_constraints()
        market_builder.set_solver('CBC')
        market_builder.dispatch(calc_prices=True)
        market = market_builder.get_market_object()

        market_overrider = historical_market_builder.MarketOverrider(market=market,
                                                                     mms_db=mms_database,
                                                                     interval=interval)

        market_overrider.set_unit_dispatch_to_historical_values()
        market_overrider.set_interconnector_flow_to_historical_values()

        market_builder.dispatch()

        market_checker = historical_market_builder.MarketChecker(market=market,
                                                                 mms_db=mms_database,
                                                                 xml_cache=xml_cache,
                                                                 interval=interval)
        assert market_checker.is_generic_constraint_slack_correct()
        assert market_checker.is_fcas_constraint_slack_correct()
        assert market_checker.is_regional_demand_meet()


def test_hist_dispatch_values_meet_demand():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)

    for interval in get_test_intervals(number=100):
        raw_inputs_loader.set_interval(interval)
        unit_inputs = units.UnitData(raw_inputs_loader)
        interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
        constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
        demand_inputs = demand.DemandData(raw_inputs_loader)

        market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                     interconnector_inputs=interconnector_inputs,
                                                                     constraint_inputs=constraint_inputs,
                                                                     demand_inputs=demand_inputs)
        market_builder.add_unit_bids_to_market()
        market_builder.add_interconnectors_to_market()
        market = market_builder.get_market_object()
        market_overrider = historical_market_builder.MarketOverrider(market=market,
                                                                     mms_db=mms_database,
                                                                     interval=interval)
        market_overrider.set_unit_dispatch_to_historical_values()
        market_overrider.set_interconnector_flow_to_historical_values()
        market_builder.dispatch()
        market_checker = historical_market_builder.MarketChecker(market=market,
                                                                 mms_db=mms_database,
                                                                 xml_cache=xml_cache,
                                                                 interval=interval)
        test_passed = market_checker.is_regional_demand_meet()
        assert test_passed
    con.close()


def test_against_10_interval_benchmark():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)
    outputs = []
    for interval in get_test_intervals(number=10):
        raw_inputs_loader.set_interval(interval)
        unit_inputs = units.UnitData(raw_inputs_loader)
        interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
        constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
        demand_inputs = demand.DemandData(raw_inputs_loader)

        market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                     interconnector_inputs=interconnector_inputs,
                                                                     constraint_inputs=constraint_inputs,
                                                                     demand_inputs=demand_inputs)
        market_builder.add_unit_bids_to_market()
        market_builder.add_interconnectors_to_market()
        market_builder.add_generic_constraints_with_fcas_requirements_interface()
        market_builder.set_unit_fcas_constraints()
        market_builder.set_unit_limit_constraints()
        market_builder.set_region_demand_constraints()
        market_builder.set_ramp_rate_limits()
        market_builder.set_fast_start_constraints()
        market_builder.set_solver('CBC')
        market_builder.dispatch(calc_prices=True)
        market = market_builder.get_market_object()

        market_checker = historical_market_builder.MarketChecker(market=market,
                                                                 mms_db=mms_database,
                                                                 xml_cache=xml_cache,
                                                                 interval=interval)
        price_comp = market_checker.get_price_comparison()
        outputs.append(price_comp)
    outputs = pd.concat(outputs)
    outputs.to_csv('latest_10_interval_run.csv', index=False)
    benchmark = pd.read_csv('10_interval_benchmark.csv')
    assert_frame_equal(outputs.reset_index(drop=True), benchmark, check_exact=False, atol=5e-2)


def test_against_100_interval_benchmark():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)
    outputs = []
    for interval in get_test_intervals(number=100):
        raw_inputs_loader.set_interval(interval)
        unit_inputs = units.UnitData(raw_inputs_loader)
        interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
        constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
        demand_inputs = demand.DemandData(raw_inputs_loader)

        market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                     interconnector_inputs=interconnector_inputs,
                                                                     constraint_inputs=constraint_inputs,
                                                                     demand_inputs=demand_inputs)
        market_builder.add_unit_bids_to_market()
        market_builder.add_interconnectors_to_market()
        market_builder.add_generic_constraints_with_fcas_requirements_interface()
        market_builder.set_unit_fcas_constraints()
        market_builder.set_unit_limit_constraints()
        market_builder.set_region_demand_constraints()
        market_builder.set_ramp_rate_limits()
        market_builder.set_fast_start_constraints()
        market_builder.set_solver('CBC')
        market_builder.dispatch(calc_prices=True)
        market = market_builder.get_market_object()

        market_checker = historical_market_builder.MarketChecker(market=market,
                                                                 mms_db=mms_database,
                                                                 xml_cache=xml_cache,
                                                                 interval=interval)
        price_comp = market_checker.get_price_comparison()
        outputs.append(price_comp)

    outputs = pd.concat(outputs)
    outputs.to_csv('latest_100_interval_run.csv', index=False)
    benchmark = pd.read_csv('100_interval_benchmark.csv')
    assert_frame_equal(outputs.reset_index(drop=True), benchmark, check_exact=False, atol=1e-2)


def test_against_1000_interval_benchmark():
    con = sqlite3.connect(test_db)
    mms_database = mms_db.DBManager(con)
    xml_cache_manager = xml_cache.XMLCacheManager(test_xml_cache)
    raw_inputs_loader = loaders.RawInputsLoader(nemde_xml_cache_manager=xml_cache_manager,
                                                market_management_system_database=mms_database)
    outputs = []
    for interval in get_test_intervals(number=1000):
        raw_inputs_loader.set_interval(interval)
        unit_inputs = units.UnitData(raw_inputs_loader)
        interconnector_inputs = interconnectors.InterconnectorData(raw_inputs_loader)
        constraint_inputs = constraints.ConstraintData(raw_inputs_loader)
        demand_inputs = demand.DemandData(raw_inputs_loader)

        market_builder = historical_market_builder.SpotMarketBuilder(unit_inputs=unit_inputs,
                                                                     interconnector_inputs=interconnector_inputs,
                                                                     constraint_inputs=constraint_inputs,
                                                                     demand_inputs=demand_inputs)
        market_builder.add_unit_bids_to_market()
        market_builder.add_interconnectors_to_market()
        market_builder.add_generic_constraints_with_fcas_requirements_interface()
        market_builder.set_unit_fcas_constraints()
        market_builder.set_unit_limit_constraints()
        market_builder.set_region_demand_constraints()
        market_builder.set_ramp_rate_limits()
        market_builder.set_fast_start_constraints()
        market_builder.set_solver('CBC')
        market_builder.dispatch(calc_prices=True)
        market = market_builder.get_market_object()

        market_checker = historical_market_builder.MarketChecker(market=market,
                                                                 mms_db=mms_database,
                                                                 xml_cache=xml_cache,
                                                                 interval=interval)
        price_comp = market_checker.get_price_comparison()
        outputs.append(price_comp)

    outputs = pd.concat(outputs)
    outputs.to_csv('latest_1000_interval_run.csv', index=False)
    benchmark = pd.read_csv('1000_interval_benchmark.csv')
    assert_frame_equal(outputs.reset_index(drop=True), benchmark.reset_index(drop=True), check_less_precise=3)

================
File: tests\test_historical_spot_market_inputs.py
================
import pytest
import pandas as pd
import subprocess
import shlex
import time
import platform
from pandas._testing import assert_frame_equal
from nempy.historical_inputs import mms_db


def test_download_to_df():
    if platform.system() == 'Windows':
        server = subprocess.Popen(shlex.split('python -m http.server 8080 --bind 127.0.0.1'))
    else:
        server = subprocess.Popen(shlex.split('python3 -m http.server 8080 --bind 127.0.0.1'))
    try:
        time.sleep(1)
        output_1 = mms_db._download_to_df(
            url='http://127.0.0.1:8080/tests/test_files/{table}_{year}{month}01.zip',
            table_name='table_one', year=2020, month=1)
        expected_1 = pd.DataFrame({
            'a': [1, 2],
            'b': [4, 5]
        })

        output_2 = mms_db._download_to_df(
            url='http://127.0.0.1:8080/tests/test_files/{table}_{year}{month}01.zip',
            table_name='table_two', year=2019, month=2)
        expected_2 = pd.DataFrame({
            'c': [1, 2],
            'd': [4, 5]
        })
    finally:
        server.terminate()
    assert_frame_equal(output_1, expected_1)
    assert_frame_equal(output_2, expected_2)


def test_download_to_df_raises_on_missing_data():
    if platform.system() == 'Windows':
        server = subprocess.Popen(shlex.split('python -m http.server 8080 --bind 127.0.0.1'))
    else:
        server = subprocess.Popen(shlex.split('python3 -m http.server 8080 --bind 127.0.0.1'))
    time.sleep(1)
    try:
        with pytest.raises(mms_db._MissingData) as exc_info:
            mms_db._download_to_df(
                url='http://127.0.0.1:8080/tests/test_files/{table}_{year}{month}01.zip',
                table_name='table_two', year=2019, month=3)
    finally:
        server.terminate()


def test_download_to_df_raises_on_data_not_on_nemweb():
    with pytest.raises(mms_db._MissingData):
        url = ('http://nemweb.com.au/Data_Archive/Wholesale_Electricity/MMSDM/{year}/MMSDM_{year}_{month}/' +
               'MMSDM_Historical_Data_SQLLoader/DATA/PUBLIC_DVD_{table}_{year}{month}010000.zip')
        mms_db._download_to_df(
            url=url, table_name='DISPATCHREGIONSUM', year=2050, month=3)

================
File: tests\test_interconnector_loss_functions.py
================
import pytest
import pandas as pd
from nempy.historical_inputs import interconnectors


def test_create_loss_function():
    # Interconnector flow
    flow = 600.0

    # Arbitrary demand inputs
    nsw_demand = 7000.0
    qld_demand = 5000.0
    demand = pd.DataFrame({
        'region': ['NSW1', 'QLD1'],
        'loss_function_demand': [nsw_demand, qld_demand]
    })

    # Loss model details from 2020 Jan NEM web files.
    demand_coefficients = pd.DataFrame({
        'interconnector': ['NSW1-QLD1', 'NSW1-QLD1'],
        'region': ['NSW1', 'QLD1'],
        'demand_coefficient': [-0.00000035146, 0.000010044]
    })

    # Loss model details from 2020 Jan NEM web files.
    interconnector_coefficients = pd.DataFrame({
        'interconnector': ['NSW1-QLD1'],
        'loss_constant': [0.9529],
        'flow_coefficient': [0.00019617],
        'from_region_loss_share': [0.5]
    })

    loss_function = interconnectors.create_loss_functions(interconnector_coefficients,
                                                          demand_coefficients, demand)

    output_losses = loss_function['loss_function'].loc[0](flow)

    expected_losses = (-0.0471 - 3.5146E-07 * nsw_demand + 1.0044E-05 * qld_demand) * flow + 9.8083E-05 * flow ** 2

    assert(pytest.approx(expected_losses, 0.0001) == output_losses)


def test_create_loss_function_vic_nsw():
    # Interconnector flow
    flow = 600.0

    # Arbitrary demand inputs
    vic_demand = 6000.0
    nsw_demand = 7000.0
    sa_demand = 3000.0
    demand = pd.DataFrame({
        'region': ['NSW1', 'VIC1', 'SA1'],
        'loss_function_demand': [nsw_demand, vic_demand, sa_demand]
    })

    # Loss model details from 2020 Jan NEM web files.
    demand_coefficients = pd.DataFrame({
        'interconnector': ['VIC1-NSW1', 'VIC1-NSW1', 'VIC1-NSW1'],
        'region': ['NSW1', 'VIC1', 'SA1'],
        'demand_coefficient': [0.000021734, -0.000031523, -0.000065967]
    })

    # Loss model details from 2020 Jan NEM web files.
    interconnector_coefficients = pd.DataFrame({
        'interconnector': ['VIC1-NSW1'],
        'loss_constant': [1.0657],
        'flow_coefficient': [0.00017027],
        'from_region_loss_share': [0.5]
    })

    loss_function = interconnectors.create_loss_functions(interconnector_coefficients,
                                                          demand_coefficients, demand)

    output_losses = loss_function['loss_function'].loc[0](flow)

    expected_losses = (0.0657 - 3.1523E-05 * vic_demand + 2.1734E-05 * nsw_demand
                       - 6.5967E-05 * sa_demand) * flow + 8.5133E-05 * flow ** 2

    assert (pytest.approx(expected_losses, 0.0001) == output_losses)


def test_create_loss_function_bass_link():
    # Interconnector flow
    flow = -433.0

    # Arbitrary demand inputs
    nsw_demand = 6000.0
    qld_demand = 7000.0
    demand = pd.DataFrame({
        'region': ['VIC1', 'QLD1'],
        'loss_function_demand': [nsw_demand, qld_demand]
    })

    # Loss model details from 2020 Jan NEM web files.
    demand_coefficients = pd.DataFrame({
        'interconnector': ['BL'],
        'region': ['VIC1'],
        'demand_coefficient': [0.0]
    })

    # Loss model details from 2020 Jan NEM web files.
    interconnector_coefficients = pd.DataFrame({
        'interconnector': ['BL'],
        'loss_constant': [0.99608],
        'flow_coefficient': [0.00020786],
        'from_region_loss_share': [0.5]
    })

    loss_function = interconnectors.create_loss_functions(interconnector_coefficients,
                                                          demand_coefficients, demand)

    output_losses = loss_function['loss_function'].loc[0](flow)

    expected_losses = (-3.92E-3) * flow + (1.0393E-4) * flow ** 2

    assert(pytest.approx(expected_losses, 0.0001) == output_losses)

================
File: tests\test_markets.py
================
import pandas as pd
from pandas._testing import assert_frame_equal
from nempy import markets


def test_one_region_energy_market():
    # Volume of each bid, number of bid bands must equal number of bands in price_bids.
    volume_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [20.0, 20.0],  # MW
        '2': [50.0, 30.0],  # MW
    })

    # Price of each bid, bids must be monotonically increasing.
    price_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [50.0, 52.0],  # $/MW
        '2': [53.0, 60.0],  # $/MW
    })

    # Factors limiting unit output
    unit_limits = pd.DataFrame({
        'unit': ['A', 'B'],
        'capacity': [55.0, 90.0],  # MW
    })

    # Other unit properties
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'region': ['NSW', 'NSW'],  # MW
        'loss_factor': [0.9, 0.95]  # MW/h
    })

    demand = pd.DataFrame({
        'region': ['NSW'],
        'demand': [60.0]  # MW
    })

    market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW'])
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_bid_capacity_constraints(unit_limits)
    market.set_unit_price_bids(price_bids)
    market.set_demand_constraints(demand)
    market.dispatch()

    expected_prices = pd.DataFrame({
        'region': ['NSW'],
        'price': [53 / 0.9]
    })

    expected_dispatch = pd.DataFrame({
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'dispatch': [40.0, 20.0]
    })

    assert_frame_equal(market.get_energy_prices(), expected_prices)
    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)


def test_two_region_energy_market():
    # Volume of each bid, number of bid bands must equal number of bands in price_bids.
    volume_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [20.0, 20.0],  # MW
        '2': [50.0, 100.0],  # MW
    })

    # Price of each bid, bids must be monotonically increasing.
    price_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [50.0, 52.0],  # $/MW
        '2': [53.0, 60.0],  # $/MW
    })

    # Factors limiting unit output
    unit_limits = pd.DataFrame({
        'unit': ['A', 'B'],
        'capacity': [70.0, 120.0],  # MW
    })

    # Other unit properties
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'region': ['NSW', 'VIC'],  # MW
        'loss_factor': [0.9, 0.95]  # MW/h
    })

    demand = pd.DataFrame({
        'region': ['NSW', 'VIC'],
        'demand': [60.0, 80.0]  # MW
    })

    market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW', 'VIC'])
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_bid_capacity_constraints(unit_limits)
    market.set_unit_price_bids(price_bids)
    market.set_demand_constraints(demand)
    market.dispatch()

    expected_prices = pd.DataFrame({
        'region': ['NSW', 'VIC'],
        'price': [53 / 0.9, 60 / 0.95]
    })

    expected_dispatch = pd.DataFrame({
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'dispatch': [60.0, 80.0]
    })

    assert_frame_equal(market.get_energy_prices(), expected_prices)
    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)


def test_one_interconnector():
    # The only generator is located in NSW.
    unit_info = pd.DataFrame({
        'unit': ['A'],
        'region': ['NSW']  # MW
    })

    # Create a market instance.
    market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW', 'VIC'])

    # Volume of each bids.
    volume_bids = pd.DataFrame({
        'unit': ['A'],
        '1': [100.0]  # MW
    })

    market.set_unit_volume_bids(volume_bids)

    # Price of each bid.
    price_bids = pd.DataFrame({
        'unit': ['A'],
        '1': [50.0]  # $/MW
    })

    market.set_unit_price_bids(price_bids)

    # NSW has no demand but VIC has 90 MW.
    demand = pd.DataFrame({
        'region': ['NSW', 'VIC'],
        'demand': [0.0, 90.0]  # MW
    })

    market.set_demand_constraints(demand)

    # There is one interconnector between NSW and VIC. Its nominal direction is towards VIC.
    interconnectors = pd.DataFrame({
        'interconnector': ['little_link'],
        'to_region': ['VIC'],
        'from_region': ['NSW'],
        'max': [100.0],
        'min': [-120.0]
    })

    market.set_interconnectors(interconnectors)

    # The interconnector loss function. In this case losses are always 5 % of line flow.
    def constant_losses(flow):
        return abs(flow) * 0.05

    # The loss function on a per interconnector basis. Also details how the losses should be proportioned to the
    # connected regions.
    loss_functions = pd.DataFrame({
        'interconnector': ['little_link'],
        'from_region_loss_share': [0.5],  # losses are shared equally.
        'loss_function': [constant_losses]
    })

    # The points to linearly interpolate the loss function bewtween. In this example the loss function is linear so only
    # three points are needed, but if a non linear loss function was used then more points would be better.
    interpolation_break_points = pd.DataFrame({
        'interconnector': ['little_link', 'little_link', 'little_link'],
        'loss_segment': [1, 2, 3],
        'break_point': [-120.0, 0.0, 100]
    })

    market.set_interconnector_losses(loss_functions, interpolation_break_points)

    # Calculate dispatch.
    market.dispatch()

    expected_prices = pd.DataFrame({
        'region': ['NSW', 'VIC'],
        'price': [50., 50. * (((90. / 0.975) + (90. / 0.975) * 0.025) / 90)]
    })

    expected_dispatch = pd.DataFrame({
        'unit': ['A'],
        'service': ['energy'],
        'dispatch': [(90.0 / 0.975) + (90.0 / 0.975) * 0.025]
    })

    expected_interconnector_flow = pd.DataFrame({
        'interconnector': ['little_link'],
        'link': ['little_link'],
        'flow': [90.0/0.975],
        'losses': [(90.0/0.975) * 0.05]
    })

    assert_frame_equal(market.get_energy_prices(), expected_prices)
    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)
    assert_frame_equal(market.get_interconnector_flows(), expected_interconnector_flow)


def test_one_region_energy_and_raise_regulation_markets():
    # Volume of each bid, number of bands must equal number of bands in price_bids.
    volume_bids = pd.DataFrame({
        'unit': ['A', 'B', 'B'],
        'service': ['energy', 'energy', 'raise_reg'],
        '1': [100.0, 110.0, 15.0],  # MW
    })

    # Price of each bid, bids must be monotonically increasing.
    price_bids = pd.DataFrame({
        'unit': ['A', 'B', 'B'],
        'service': ['energy', 'energy', 'raise_reg'],
        '1': [50.0, 60.0, 20.0],  # $/MW
    })

    # Participant defined operational constraints on FCAS enablement.
    fcas_trapeziums = pd.DataFrame({
        'unit': ['B'],
        'service': ['raise_reg'],
        'max_availability': [15.0],
        'enablement_min': [50.0],
        'low_break_point': [65.0],
        'high_break_point': [95.0],
        'enablement_max': [110.0]
    })

    # Unit locations.
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'region': ['NSW', 'NSW']
    })

    # The demand in the region\s being dispatched.
    demand = pd.DataFrame({
        'region': ['NSW'],
        'demand': [100.0]  # MW
    })

    # FCAS requirement in the region\s being dispatched.
    fcas_requirement = pd.DataFrame({
        'set': ['nsw_regulation_requirement'],
        'region': ['NSW'],
        'service': ['raise_reg'],
        'volume': [10.0]  # MW
    })

    # Create the market model
    market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW'])
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_price_bids(price_bids)
    market.set_fcas_max_availability(
        fcas_trapeziums.loc[:, ['unit', 'service', 'max_availability']])
    market.set_energy_and_regulation_capacity_constraints(fcas_trapeziums)
    market.set_demand_constraints(demand)
    market.set_fcas_requirements_constraints(fcas_requirement)

    # Calculate dispatch and pricing
    market.dispatch()

    # Return the total dispatch of each unit in MW. Note that despite the energy bid of A being cheaper and capable
    # of meeting total demand, the more expensive B is dispatch up to 60 MW so that it can deliver the raise_reg
    # service.
    print(market.get_unit_dispatch())
    #   unit    service  dispatch
    # 0    A     energy      40.0
    # 1    B     energy      60.0
    # 2    B  raise_reg      10.0

    # Return the price of energy.
    print(market.get_energy_prices())
    #   region  price
    # 0    NSW   60.0

    # Return the price of regulation FCAS. Note to meet marginal FCAS demand unit B has to turn up and unit A needs to
    # turn down, at a net marginal cost of 10 $/MW, it would also cost 20 $/MW to increase unit B FCAS dispatch, hence
    # the total marginal cost of raise reg is 10 + 20 = 30.
    print(market.get_fcas_prices())
    #                           set  price
    # 0  nsw_regulation_requirement   30.0

    expected_dispatch = pd.DataFrame({
        'unit': ['A', 'B', 'B'],
        'service': ['energy', 'energy', 'raise_reg'],
        'dispatch': [40.0, 60.0, 10.0]
    })

    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)

    expected_energy_prices = pd.DataFrame({
        'region': ['NSW'],
        'price': [50.0]
    })

    assert_frame_equal(market.get_energy_prices(), expected_energy_prices)

    expected_fcas_prices = pd.DataFrame({
        'region': ['NSW'],
        'service': ['raise_reg'],
        'price': [30.0]
    })

    assert_frame_equal(market.get_fcas_prices(), expected_fcas_prices)


def test_raise_6s_and_raise_reg():
    # Volume of each bid.
    volume_bids = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B', 'B'],
        'service': ['energy', 'raise_6s', 'energy', 'raise_6s', 'raise_reg'],
        '1': [100.0, 10.0, 110.0, 15.0, 15.0],  # MW
    })

    # Price of each bid.
    price_bids = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B', 'B'],
        'service': ['energy', 'raise_6s', 'energy', 'raise_6s', 'raise_reg'],
        '1': [50.0, 35.0, 60.0, 20.0, 30.0],  # $/MW
    })

    # Participant defined operational constraints on FCAS enablement.
    fcas_trapeziums = pd.DataFrame({
        'unit': ['B', 'B', 'A'],
        'service': ['raise_reg', 'raise_6s', 'raise_6s'],
        'max_availability': [15.0, 15.0, 10.0],
        'enablement_min': [50.0, 50.0, 70.0],
        'low_break_point': [65.0, 65.0, 80.0],
        'high_break_point': [95.0, 95.0, 100.0],
        'enablement_max': [110.0, 110.0, 110.0]
    })

    # Unit locations.
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'region': ['NSW', 'NSW']
    })

    # The demand in the region\s being dispatched.
    demand = pd.DataFrame({
        'region': ['NSW'],
        'demand': [195.0]  # MW
    })

    # FCAS requirement in the region\s being dispatched.
    fcas_requirements = pd.DataFrame({
        'set': ['nsw_regulation_requirement', 'nsw_raise_6s_requirement'],
        'region': ['NSW', 'NSW'],
        'service': ['raise_reg', 'raise_6s'],
        'volume': [10.0, 10.0]  # MW
    })

    # Create the market model with unit service bids.
    market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW'])
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_price_bids(price_bids)

    # Create constraints that enforce the top of the FCAS trapezium.
    fcas_availability = fcas_trapeziums.loc[:, ['unit', 'service', 'max_availability']]
    market.set_fcas_max_availability(fcas_availability)

    # Create constraints the enforce the lower and upper slope of the FCAS regulation
    # service trapeziums.
    regulation_trapeziums = fcas_trapeziums[fcas_trapeziums['service'] == 'raise_reg']
    market.set_energy_and_regulation_capacity_constraints(regulation_trapeziums)

    # Create constraints that enforce the lower and upper slope of the FCAS contingency
    # trapezium. These constrains also scale slopes of the trapezium to ensure the
    # co-dispatch of contingency and regulation services is technically feasible.
    contingency_trapeziums = fcas_trapeziums[fcas_trapeziums['service'] == 'raise_6s']
    market.set_joint_capacity_constraints(contingency_trapeziums)

    # Set the demand for energy.
    market.set_demand_constraints(demand)

    # Set the required volume of FCAS services.
    market.set_fcas_requirements_constraints(fcas_requirements)

    # Calculate dispatch and pricing
    market.dispatch()

    expected_dispatch = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B', 'B'],
        'service': ['energy', 'raise_6s', 'energy', 'raise_6s', 'raise_reg'],
        'dispatch': [100.0, 5.0, 95.0, 5.0, 10.0]
    })

    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)

    expected_energy_prices = pd.DataFrame({
        'region': ['NSW'],
        'price': [75.0]
    })

    assert_frame_equal(market.get_energy_prices(), expected_energy_prices)

    expected_fcas_prices = pd.DataFrame({
        'region': ['NSW', 'NSW'],
        'service': ['raise_6s', 'raise_reg'],
        'price': [35.0, 45.0]
    })

    assert_frame_equal(market.get_fcas_prices(), expected_fcas_prices)


def test_raise_1s_and_raise_reg():
    # Volume of each bid.
    volume_bids = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B', 'B'],
        'service': ['energy', 'raise_1s', 'energy', 'raise_1s', 'raise_reg'],
        '1': [100.0, 10.0, 110.0, 15.0, 15.0],  # MW
    })

    # Price of each bid.
    price_bids = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B', 'B'],
        'service': ['energy', 'raise_1s', 'energy', 'raise_1s', 'raise_reg'],
        '1': [50.0, 35.0, 60.0, 20.0, 30.0],  # $/MW
    })

    # Participant defined operational constraints on FCAS enablement.
    fcas_trapeziums = pd.DataFrame({
        'unit': ['B', 'B', 'A'],
        'service': ['raise_reg', 'raise_1s', 'raise_1s'],
        'max_availability': [15.0, 15.0, 10.0],
        'enablement_min': [50.0, 50.0, 70.0],
        'low_break_point': [65.0, 65.0, 80.0],
        'high_break_point': [95.0, 95.0, 100.0],
        'enablement_max': [110.0, 110.0, 110.0]
    })

    # Unit locations.
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'region': ['NSW', 'NSW']
    })

    # The demand in the region\s being dispatched.
    demand = pd.DataFrame({
        'region': ['NSW'],
        'demand': [195.0]  # MW
    })

    # FCAS requirement in the region\s being dispatched.
    fcas_requirements = pd.DataFrame({
        'set': ['nsw_regulation_requirement', 'nsw_raise_1s_requirement'],
        'region': ['NSW', 'NSW'],
        'service': ['raise_reg', 'raise_1s'],
        'volume': [10.0, 10.0]  # MW
    })

    # Create the market model with unit service bids.
    market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW'])
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_price_bids(price_bids)

    # Create constraints that enforce the top of the FCAS trapezium.
    fcas_availability = fcas_trapeziums.loc[:, ['unit', 'service', 'max_availability']]
    market.set_fcas_max_availability(fcas_availability)

    # Create constraints the enforce the lower and upper slope of the FCAS regulation
    # service trapeziums.
    regulation_trapeziums = fcas_trapeziums[fcas_trapeziums['service'] == 'raise_reg']
    market.set_energy_and_regulation_capacity_constraints(regulation_trapeziums)

    # Create constraints that enforce the lower and upper slope of the FCAS contingency
    # trapezium. These constrains also scale slopes of the trapezium to ensure the
    # co-dispatch of contingency and regulation services is technically feasible.
    contingency_trapeziums = fcas_trapeziums[fcas_trapeziums['service'] == 'raise_1s']
    market.set_joint_capacity_constraints(contingency_trapeziums)

    # Set the demand for energy.
    market.set_demand_constraints(demand)

    # Set the required volume of FCAS services.
    market.set_fcas_requirements_constraints(fcas_requirements)

    # Calculate dispatch and pricing
    market.dispatch()

    expected_dispatch = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B', 'B'],
        'service': ['energy', 'raise_1s', 'energy', 'raise_1s', 'raise_reg'],
        'dispatch': [100.0, 5.0, 95.0, 5.0, 10.0]
    })

    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)

    expected_energy_prices = pd.DataFrame({
        'region': ['NSW'],
        'price': [75.0]
    })

    assert_frame_equal(market.get_energy_prices(), expected_energy_prices)

    expected_fcas_prices = pd.DataFrame({
        'region': ['NSW', 'NSW'],
        'service': ['raise_1s', 'raise_reg'],
        'price': [35.0, 45.0]
    })

    assert_frame_equal(market.get_fcas_prices(), expected_fcas_prices)


def test_lower_1s_and_lower_reg():
    # Volume of each bid.
    volume_bids = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B', 'B'],
        'service': ['energy', 'lower_1s', 'energy', 'lower_1s', 'lower_reg'],
        '1': [100.0, 10.0, 110.0, 15.0, 15.0],  # MW
    })

    # Price of each bid.
    price_bids = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B', 'B'],
        'service': ['energy', 'lower_1s', 'energy', 'lower_1s', 'lower_reg'],
        '1': [50.0, 35.0, 60.0, 20.0, 30.0],  # $/MW
    })

    # Participant defined operational constraints on FCAS enablement.
    fcas_trapeziums = pd.DataFrame({
        'unit': ['B', 'B', 'A'],
        'service': ['lower_reg', 'lower_1s', 'lower_1s'],
        'max_availability': [15.0, 15.0, 10.0],
        'enablement_min': [50.0, 50.0, 70.0],
        'low_break_point': [65.0, 65.0, 80.0],
        'high_break_point': [95.0, 95.0, 100.0],
        'enablement_max': [110.0, 110.0, 110.0]
    })

    # Unit locations.
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'region': ['NSW', 'NSW']
    })

    # The demand in the region\s being dispatched.
    demand = pd.DataFrame({
        'region': ['NSW'],
        'demand': [195.0]  # MW
    })

    # FCAS requirement in the region\s being dispatched.
    fcas_requirements = pd.DataFrame({
        'set': ['nsw_regulation_requirement', 'nsw_lower_1s_requirement'],
        'region': ['NSW', 'NSW'],
        'service': ['lower_reg', 'lower_1s'],
        'volume': [10.0, 10.0]  # MW
    })

    # Create the market model with unit service bids.
    market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW'])
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_price_bids(price_bids)

    # Create constraints that enforce the top of the FCAS trapezium.
    fcas_availability = fcas_trapeziums.loc[:, ['unit', 'service', 'max_availability']]
    market.set_fcas_max_availability(fcas_availability)

    # Create constraints the enforce the lower and upper slope of the FCAS regulation
    # service trapeziums.
    regulation_trapeziums = fcas_trapeziums[fcas_trapeziums['service'] == 'lower_reg']
    market.set_energy_and_regulation_capacity_constraints(regulation_trapeziums)

    # Create constraints that enforce the lower and upper slope of the FCAS contingency
    # trapezium. These constrains also scale slopes of the trapezium to ensure the
    # co-dispatch of contingency and regulation services is technically feasible.
    contingency_trapeziums = fcas_trapeziums[fcas_trapeziums['service'] == 'lower_1s']
    market.set_joint_capacity_constraints(contingency_trapeziums)

    # Set the demand for energy.
    market.set_demand_constraints(demand)

    # Set the required volume of FCAS services.
    market.set_fcas_requirements_constraints(fcas_requirements)

    # Calculate dispatch and pricing
    market.dispatch()

    expected_dispatch = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B', 'B'],
        'service': ['energy', 'lower_1s', 'energy', 'lower_1s', 'lower_reg'],
        'dispatch': [100.0, 0.0, 95.0, 10.0, 10.0]
    })

    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)

    expected_energy_prices = pd.DataFrame({
        'region': ['NSW'],
        'price': [60.0]
    })

    assert_frame_equal(market.get_energy_prices(), expected_energy_prices)

    expected_fcas_prices = pd.DataFrame({
        'region': ['NSW', 'NSW'],
        'service': ['lower_1s', 'lower_reg'],
        'price': [20.0, 30.0]
    })

    assert_frame_equal(market.get_fcas_prices(), expected_fcas_prices)


def test_two_region_energy_market_with_regional_generic_constraints():
    # Volume of each bid, number of bid bands must equal number of bands in price_bids.
    volume_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [100.0, 100.0]
    })

    # Price of each bid, bids must be monotonically increasing.
    price_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [50.0, 20.0]
    })

    # Factors limiting unit output
    unit_limits = pd.DataFrame({
        'unit': ['A', 'B'],
        'capacity': [100.0, 120.0],  # MW
    })

    # Other unit properties
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'region': ['NSW', 'VIC']
    })

    demand = pd.DataFrame({
        'region': ['NSW', 'VIC'],
        'demand': [60.0, 80.0]  # MW
    })

    # Generic constraints
    generic_cons = pd.DataFrame({
        'set': ['X'],
        'type': ['>='],
        'rhs': [65.0],
    })

    region_coefficients = pd.DataFrame({
        'set': ['X'],
        'region': ['NSW'],
        'service': ['energy'],
        'coefficient': [1.0]
    })

    # There is one interconnector between NSW and VIC. Its nominal direction is towards VIC.
    interconnectors = pd.DataFrame({
        'interconnector': ['little_link'],
        'to_region': ['VIC'],
        'from_region': ['NSW'],
        'max': [100.0],
        'min': [-120.0]
    })

    market = markets.SpotMarket(unit_info=unit_info, market_regions=['VIC', 'NSW'])
    market.set_interconnectors(interconnectors)
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_bid_capacity_constraints(unit_limits)
    market.set_unit_price_bids(price_bids)
    market.set_demand_constraints(demand)
    market.set_generic_constraints(generic_cons)
    market.link_regions_to_generic_constraints(region_coefficients)
    market.dispatch()

    expected_prices = pd.DataFrame({
        'region': ['NSW', 'VIC'],
        'price': [20.0, 20.0]
    })

    expected_dispatch = pd.DataFrame({
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'dispatch': [65.0, 75.0]
    })

    assert_frame_equal(market.get_energy_prices(), expected_prices)
    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)


def test_two_region_energy_market_with_unit_generic_constraints():
    # Volume of each bid, number of bid bands must equal number of bands in price_bids.
    volume_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [100.0, 100.0]
    })

    # Price of each bid, bids must be monotonically increasing.
    price_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [50.0, 20.0]
    })

    # Factors limiting unit output
    unit_limits = pd.DataFrame({
        'unit': ['A', 'B'],
        'capacity': [100.0, 120.0],  # MW
    })

    # Other unit properties
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'region': ['NSW', 'VIC']
    })

    demand = pd.DataFrame({
        'region': ['NSW', 'VIC'],
        'demand': [60.0, 80.0]  # MW
    })

    # Generic constraints
    generic_cons = pd.DataFrame({
        'set': ['X'],
        'type': ['>='],
        'rhs': [65.0],
    })

    unit_coefficients = pd.DataFrame({
        'set': ['X'],
        'unit': ['A'],
        'service': ['energy'],
        'coefficient': [1.0]
    })

    # There is one interconnector between NSW and VIC. Its nominal direction is towards VIC.
    interconnectors = pd.DataFrame({
        'interconnector': ['little_link'],
        'to_region': ['VIC'],
        'from_region': ['NSW'],
        'max': [100.0],
        'min': [-120.0]
    })

    market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW', 'VIC'])
    market.set_interconnectors(interconnectors)
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_bid_capacity_constraints(unit_limits)
    market.set_unit_price_bids(price_bids)
    market.set_demand_constraints(demand)
    market.set_generic_constraints(generic_cons)
    market.link_units_to_generic_constraints(unit_coefficients)
    market.dispatch()

    expected_prices = pd.DataFrame({
        'region': ['NSW', 'VIC'],
        'price': [20.0, 20.0]
    })

    expected_dispatch = pd.DataFrame({
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'dispatch': [65.0, 75.0]
    })

    assert_frame_equal(market.get_energy_prices(), expected_prices)
    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)


def test_two_region_energy_market_with_interconnector_generic_constraints():
    # Volume of each bid, number of bid bands must equal number of bands in price_bids.
    volume_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [100.0, 100.0]
    })

    # Price of each bid, bids must be monotonically increasing.
    price_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [50.0, 20.0]
    })

    # Factors limiting unit output
    unit_limits = pd.DataFrame({
        'unit': ['A', 'B'],
        'capacity': [100.0, 120.0],  # MW
    })

    # Other unit properties
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'region': ['NSW', 'VIC']
    })

    demand = pd.DataFrame({
        'region': ['NSW', 'VIC'],
        'demand': [60.0, 80.0]  # MW
    })

    # Generic constraints
    generic_cons = pd.DataFrame({
        'set': ['X'],
        'type': ['>='],
        'rhs': [10.0],
    })

    interconnector_coefficients = pd.DataFrame({
        'set': ['X'],
        'interconnector': ['little_link'],
        'coefficient': [1.0]
    })

    # There is one interconnector between NSW and VIC. Its nominal direction is towards VIC.
    interconnectors = pd.DataFrame({
        'interconnector': ['little_link'],
        'to_region': ['VIC'],
        'from_region': ['NSW'],
        'max': [100.0],
        'min': [-120.0]
    })

    market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW', 'VIC'])
    market.set_interconnectors(interconnectors)
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_bid_capacity_constraints(unit_limits)
    market.set_unit_price_bids(price_bids)
    market.set_demand_constraints(demand)
    market.set_generic_constraints(generic_cons)
    market.link_interconnectors_to_generic_constraints(interconnector_coefficients)
    market.dispatch()

    expected_prices = pd.DataFrame({
        'region': ['NSW', 'VIC'],
        'price': [50.0, 20.0]
    })

    expected_dispatch = pd.DataFrame({
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'dispatch': [70.0, 70.0]
    })

    assert_frame_equal(market.get_energy_prices(), expected_prices)
    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)


def test_use_unit_generic_constraints_to_exclude_unit_from_providing_raise_reg():
    # Volume of each bid.
    volume_bids = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B', 'B'],
        'service': ['energy', 'raise_6s', 'energy', 'raise_6s', 'raise_reg'],
        '1': [100.0, 11.0, 110.0, 15.0, 15.0],  # MW
    })

    # Price of each bid.
    price_bids = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B', 'B'],
        'service': ['energy', 'raise_6s', 'energy', 'raise_6s', 'raise_reg'],
        '1': [50.0, 35.0, 60.0, 20.0, 30.0],  # $/MW
    })

    # Unit locations.
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'region': ['NSW', 'NSW']
    })

    # The demand in the region\s being dispatched.
    demand = pd.DataFrame({
        'region': ['NSW'],
        'demand': [195.0]  # MW
    })

    # FCAS requirement in the region\s being dispatched.
    fcas_requirements = pd.DataFrame({
        'set': ['nsw_regulation_requirement', 'nsw_raise_6s_requirement'],
        'region': ['NSW', 'NSW'],
        'service': ['raise_reg', 'raise_6s'],
        'volume': [10.0, 10.0]  # MW
    })

    # Generic constraints
    interconnector_coefficients = pd.DataFrame({
        'set': ['nsw_raise_6s_requirement'],
        'unit': ['B'],
        'service': ['raise_6s'],
        'coefficient': [-1.0],
    })

    # Create the market model with unit service bids.
    market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW', 'VIC'])
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_price_bids(price_bids)

    # Set the demand for energy.
    market.set_demand_constraints(demand)

    # Set the required volume of FCAS services.
    market.set_fcas_requirements_constraints(fcas_requirements)

    # Create generic constraints
    market.link_units_to_generic_constraints(interconnector_coefficients)

    # Calculate dispatch and pricing
    market.dispatch()

    expected_dispatch = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B', 'B'],
        'service': ['energy', 'raise_6s', 'energy', 'raise_6s', 'raise_reg'],
        'dispatch': [100.0, 10.0, 95.0, 0.0, 10.0]
    })

    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)


def test_one_region_energy_market_with_elastic_unit_generic_constraints():
    # Volume of each bid, number of bid bands must equal number of bands in price_bids.
    volume_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [100.0, 100.0]
    })

    # Price of each bid, bids must be monotonically increasing.
    price_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [50.0, 20.0]
    })

    # Factors limiting unit output
    unit_limits = pd.DataFrame({
        'unit': ['A', 'B'],
        'capacity': [100.0, 120.0],  # MW
    })

    # Other unit properties
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'region': ['NSW', 'NSW']
    })

    demand = pd.DataFrame({
        'region': ['NSW'],
        'demand': [80.0]  # MW
    })

    # Generic constraints
    generic_cons = pd.DataFrame({
        'set': ['X'],
        'type': ['>='],
        'rhs': [65.0],
    })

    violation_costs = pd.DataFrame({
        'set': ['X'],
        'cost': [1000.0]
    })

    unit_coefficients = pd.DataFrame({
        'set': ['X'],
        'unit': ['A'],
        'service': ['energy'],
        'coefficient': [1.0]
    })

    market = markets.SpotMarket(unit_info=unit_info, market_regions=['NSW'])
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_bid_capacity_constraints(unit_limits)
    market.set_unit_price_bids(price_bids)
    market.set_demand_constraints(demand)
    market.set_generic_constraints(generic_cons)
    market.make_constraints_elastic('generic', violation_costs)
    market.link_units_to_generic_constraints(unit_coefficients)
    market.dispatch()

    expected_prices = pd.DataFrame({
        'region': ['NSW'],
        'price': [20.0]
    })

    expected_dispatch = pd.DataFrame({
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'dispatch': [65.0, 15.0]
    })

    assert_frame_equal(market.get_energy_prices(), expected_prices)
    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)


def test_setting_constraint_on_unit_with_no_bid_volume_doesnt_raise_error():
    # Unit 'C' is on outage, energy bids all 0
    volume_bids = pd.DataFrame({
        'unit': ['A', 'B', 'C'],
        '1': [20.0, 50.0, 0.0],
        '2': [20.0, 30.0, 0.0],
        '3': [5.0, 10.0, 0.0]
    })

    price_bids = pd.DataFrame({
        'unit': ['A', 'B', 'C'],
        '1': [50.0, 50.0, 0.0],
        '2': [60.0, 55.0, 0.0],
        '3': [100.0, 80.0, 0.0]
    })

    unit_info = pd.DataFrame({
        'unit': ['A', 'B', 'C'],
        'region': ['NSW', 'NSW', 'NSW'],
    })

    # Max capacity also set to 0 for unit 'C'
    max_capacity = pd.DataFrame({
        'unit': ['A', 'B', 'C'],
        'capacity': [50.0, 100.0, 0.0],
    })

    demand = pd.DataFrame({
        'region': ['NSW'],
        'demand': [120.0]
    })

    market = markets.SpotMarket(unit_info=unit_info,
                                market_regions=['NSW'])
    market.set_unit_volume_bids(volume_bids)
    market.set_unit_price_bids(price_bids)
    market.set_demand_constraints(demand)
    market.set_unit_bid_capacity_constraints(max_capacity)

    market.dispatch()

    expected_prices = pd.DataFrame({
        'region': ['NSW'],
        'price': [60.0]
    })

    expected_dispatch = pd.DataFrame({
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'dispatch': [40.0, 80.0]
    })

    assert_frame_equal(market.get_energy_prices(), expected_prices)
    assert_frame_equal(market.get_unit_dispatch(), expected_dispatch)

================
File: tests\test_market_constraints.py
================
import pandas as pd
from pandas._testing import assert_frame_equal
from nempy.spot_markert_backend import market_constraints


def test_energy():
    demand = pd.DataFrame({
        'region': ['X', 'Y'],
        'demand': [16.0, 23.0],
    })
    expected_rhs = pd.DataFrame({
        'region': ['X', 'Y'],
        'constraint_id': [0, 1],
        'type': ['=', '='],
        'rhs': [16.0, 23.0],
    })
    expected_variable_map = pd.DataFrame({
        'constraint_id': [0, 1],
        'region': ['X', 'Y'],
        'service': ['energy', 'energy'],
        'coefficient': [1.0, 1.0]
    })
    output_rhs, output_variable_map = market_constraints.energy(demand, next_constraint_id=0)
    expected_rhs.index = list(expected_rhs.index)
    assert_frame_equal(output_rhs, expected_rhs)
    assert_frame_equal(output_variable_map, expected_variable_map)

================
File: tests\test_objective_function.py
================
import pandas as pd
from pandas._testing import assert_frame_equal
from nempy.spot_markert_backend import objective_function


def test_energy():
    bidding_ids = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B'],
        'capacity_band': ['1', '2', '1', '2'],
        'service': ['energy', 'energy', 'energy', 'energy'],
        'variable_id': [1, 2, 3, 4]
    })
    price_bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [16.0, 23.0],
        '2': [17.0, 18.0]
    })
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'dispatch_type': ['generator', 'load']
    })
    output = objective_function.bids(bidding_ids, price_bids, unit_info)
    expected = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B'],
        'capacity_band': ['1', '2', '1', '2'],
        'service': ['energy', 'energy', 'energy', 'energy'],
        'variable_id': [1, 2, 3, 4],
        'cost': [16.0, 17.0, -23.0, -18.0],
        'dispatch_type': ['generator', 'generator', 'load', 'load']
    })
    expected.index = list(expected.index)
    assert_frame_equal(output, expected)

================
File: tests\test_rpn_calc.py
================
from nempy.historical_inputs.rhs_calculator import _rpn_calc, _rpn_stack


def test_no_rpn_operators():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.2"""

    equation = [
        {'@TermID': '1', '@SpdID': 'BW01.NBAY1', '@SpdType': 'T', '@Multiplier': '-1', '@Value': '500'},
        {'@TermID': '2', '@SpdID': 'BW01.NBAY1', '@SpdType': 'I', '@Multiplier': '0.5', '@Value': '-1000'},
        {'@TermID': '3', '@SpdID': 'BW01.NBAY1', '@SpdType': 'R', '@Multiplier': '1', '@Value': '10000'},
    ]

    assert _rpn_calc(equation) == 9000


def test_groups():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.3"""

    equation = [
        {'@TermID': '1', '@GroupTerm': '5', '@SpdID': 'NRATSE_MNDT8', '@SpdType': 'E', '@Multiplier': '1', '@Value': '1000'},
        {'@TermID': '2', '@GroupTerm': '5', '@SpdID': 'MW_MN_8', '@SpdType': 'A', '@Multiplier': '-1', '@Value': '400'},
        {'@TermID': '3', '@GroupTerm': '5', '@SpdID': 'MW_MN_16', '@SpdType': 'A', '@Multiplier': '-0.498', '@Value': '500'},
        {'@TermID': '4', '@GroupTerm': '5', '@SpdID': 'Operating_Margin', '@SpdType': 'C', '@Multiplier': '-25'},
        {'@TermID': '5', '@SpdID': 'HEADROOM', '@SpdType': 'G', '@Multiplier': '4.197'},
        {'@TermID': '6', '@SpdID': 'TALWA1.NDT13T', '@SpdType': 'T', '@Multiplier': '-1', '@Value': '250'},
    ]

    assert _rpn_calc(equation) == 1118.222


def test_top_stack_element():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.5"""

    equation = [
        {'@TermID': '1', '@SpdID': 'NRATSE_MNDT8', '@SpdType': 'E', '@Multiplier': '1', '@Value': '1000'},
        {'@TermID': '2', '@SpdID': 'MW_MN_8', '@SpdType': 'A', '@Multiplier': '-1', '@Value': '400'},
        {'@TermID': '3', '@SpdID': 'MW_MN_16', '@SpdType': 'A', '@Multiplier': '-0.498', '@Value': '500'},
        {'@TermID': '4', '@SpdID': 'Operating_Margin', '@SpdType': 'C', '@Multiplier': '-25'},
        {'@TermID': '5', '@SpdID': 'HEADROOM', '@SpdType': 'U', '@Multiplier': '4.197'},
        {'@TermID': '6', '@SpdID': 'TALWA1.NDT13T', '@SpdType': 'T', '@Multiplier': '-1', '@Value': '250'},
    ]

    assert _rpn_calc(equation) == 1118.222


def test_step_one_of_two():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.6.1"""

    equation = [
        {'@TermID': '1', '@SpdID': 'BW01.NBAY1', '@SpdType': 'T', '@Multiplier': '1', '@Operation': 'STEP',
         '@Value': '500'},
        {'@TermID': '2', '@SpdID': 'BW02.NBAY2', '@SpdType': 'T', '@Multiplier': '1', '@Operation': 'STEP',
         '@Value': '0'}
    ]

    assert _rpn_calc(equation) == 1


def test_step_two_of_two():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.6.1"""

    equation = [
        {'@TermID': '1', '@SpdID': 'CONSTANT', '@SpdType': 'C', '@Multiplier': '2'},
        {'@TermID': '2', '@SpdID': 'BW01.NBAY1', '@SpdType': 'T', '@Multiplier': '1', '@Operation': 'PUSH',
         '@Value': '400'},
        {'@TermID': '2', '@SpdID': '', '@SpdType': 'U', '@Multiplier': '500', '@Operation': 'STEP'},
        {'@TermID': '2', '@SpdID': '', '@SpdType': 'U', '@Multiplier': '1', '@Operation': 'ADD'}
    ]

    assert _rpn_calc(equation) == 502


def test_square():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.6.2"""

    equation = [
        {'@TermID': '1', '@SpdID': 'NSW1-QLD1 ', '@SpdType': 'I', '@Multiplier': '1', '@Operation': 'POW2',
         '@Value': '100'}
    ]

    assert _rpn_calc(equation) == 10000


def test_cube():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.6.3"""

    equation = [
        {'@TermID': '1', '@SpdID': 'NSW1-QLD1 ', '@SpdType': 'I', '@Multiplier': '1', '@Operation': 'POW3',
         '@Value': '100'}
    ]

    assert _rpn_calc(equation) == 1000000


def test_sqrt():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.6.4"""

    equation = [
        {'@TermID': '1', '@SpdID': 'NSW1-QLD1 ', '@SpdType': 'I', '@Multiplier': '1', '@Operation': 'SQRT',
         '@Value': '100'}
    ]

    assert _rpn_calc(equation) == 10


def test_absolute_value():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.6.5"""

    equation = [
        {'@TermID': '1', '@SpdID': 'NSW1-QLD1 ', '@SpdType': 'I', '@Multiplier': '1', '@Operation': 'ABS',
         '@Value': '-100'}
    ]

    assert _rpn_calc(equation) == 100


def test_negation():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.6.6"""

    equation = [
        {'@TermID': '1', '@SpdID': 'NSW1-QLD1', '@SpdType': 'I', '@Multiplier': '1', '@Operation': 'NEG',
         '@Value': '100'}
    ]

    assert _rpn_calc(equation) == -100


def test_add():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.7.1"""

    equation = [
        {'@TermID': '1', '@SpdID': 'MW_MN_8', '@SpdType': 'A', '@Multiplier': '1', '@Value': '100'},
        {'@TermID': '1', '@SpdID': 'MW_MN_16', '@SpdType': 'A', '@Multiplier': '2', '@Operation': 'ADD',
         '@Value': '200'}
    ]

    assert _rpn_calc(equation) == 600


def test_sub():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.7.2"""

    equation = [
        {'@TermID': '1', '@SpdID': 'MW_MN_8', '@SpdType': 'A', '@Multiplier': '1', '@Value': '100'},
        {'@TermID': '1', '@SpdID': 'MW_MN_16', '@SpdType': 'A', '@Multiplier': '2', '@Operation': 'SUB',
         '@Value': '200'}
    ]

    assert _rpn_calc(equation) == -200


def test_multiply():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.7.3"""

    equation = [
        {'@TermID': '1', '@SpdID': 'MW_MN_8', '@SpdType': 'A', '@Multiplier': '1', '@Value': '10'},
        {'@TermID': '1', '@SpdID': 'MW_MN_16', '@SpdType': 'A', '@Multiplier': '2', '@Operation': 'MUL',
         '@Value': '20'}
    ]

    assert _rpn_calc(equation) == 400


def test_divide():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.7.4"""

    equation = [
        {'@TermID': '1', '@SpdID': 'MW_MN_8', '@SpdType': 'A', '@Multiplier': '1', '@Value': '10'},
        {'@TermID': '1', '@SpdID': 'MW_MN_16', '@SpdType': 'A', '@Multiplier': '2', '@Operation': 'DIV',
         '@Value': '20'}
    ]

    assert _rpn_calc(equation) == 1.0


def test_max():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.7.5"""

    equation = [
        {'@TermID': '1', '@SpdID': 'BW01.NBAY1', '@SpdType': 'T', '@Multiplier': '1', '@Value': '500'},
        {'@TermID': '1', '@SpdID': 'BW02.NBAY2', '@SpdType': 'T', '@Multiplier': '1', '@Operation': 'MAX',
         '@Value': '650'},
        # {'@TermID': '1', '@SpdID': 'ER01.NEPS1', '@SpdType': 'U', '@Multiplier': '1', '@Operation': 'MAX',
        #  '@Value': '670'},
        # {'@TermID': '1', '@SpdID': 'TALWA1.NDT13T', '@SpdType': 'U', '@Multiplier': '1', '@Operation': 'MAX',
        #  '@Value': '250'}
    ]

    assert _rpn_calc(equation) == 650


def test_min():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.7.6"""

    equation = [
        {'@TermID': '1', '@SpdID': 'BW01.NBAY1', '@SpdType': 'T', '@Multiplier': '1', '@Value': '500'},
        {'@TermID': '1', '@SpdID': 'BW02.NBAY2', '@SpdType': 'T', '@Multiplier': '1', '@Operation': 'MIN',
         '@Value': '650'},
        # {'@TermID': '1', '@SpdID': 'ER01.NEPS1', '@SpdType': 'U', '@Multiplier': '1', '@Operation': 'MAX',
        #  '@Value': '670'},
        # {'@TermID': '1', '@SpdID': 'TALWA1.NDT13T', '@SpdType': 'U', '@Multiplier': '1', '@Operation': 'MAX',
        #  '@Value': '250'}
    ]

    assert _rpn_calc(equation) == 500


def test_push():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.8.1"""

    equation = [
        {'@TermID': '1', '@SpdID': 'NSW1-QLD1', '@SpdType': 'I', '@Multiplier': '1', '@Value': '100'},
        {'@TermID': '1', '@SpdID': 'TALWA1.NDT13T', '@SpdType': 'T', '@Multiplier': '0.5', '@Operation': 'PUSH',
         '@Value': '350'},
    ]

    assert _rpn_stack(equation) == [175.0, 100]

    assert _rpn_calc(equation) == 175.0


def test_duplicate():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.8.2"""

    equation = [
        {'@TermID': '1', '@SpdID': 'NSW1-QLD1', '@SpdType': 'I', '@Multiplier': '1', '@Value': '200'},
        {'@TermID': '1', '@SpdType': 'U', '@Multiplier': '0.5', '@Operation': 'DUP'},
    ]

    assert _rpn_stack(equation) == [100.0, 200]

    assert _rpn_calc(equation) == 100


def test_exchange():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.8.3"""

    equation = [
        {'@TermID': '1', '@SpdID': 'BW01.NBAY1', '@SpdType': 'T', '@Multiplier': '1', '@Value': '660'},
        {'@TermID': '1', '@SpdID': 'BW01.NBAY1', '@SpdType': 'T', '@Multiplier': '1', '@Operation': 'PUSH',
         '@Value': '500'},
        {'@TermID': '1', '@SpdType': 'U', '@Multiplier': '2', '@Operation': 'EXCH'},
    ]

    assert _rpn_stack(equation) == [1320.0, 500]

    assert _rpn_calc(equation) == 1320


def test_roll_stack_down():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.8.4"""

    equation = [
        {'@TermID': '1', '@SpdID': 'BW01.NBAY1', '@SpdType': 'T', '@Multiplier': '1', '@Value': '660'},
        {'@TermID': '1', '@SpdID': 'BW01.NBAY2', '@SpdType': 'T', '@Multiplier': '1', '@Operation': 'PUSH',
         '@Value': '550'},
        {'@TermID': '1', '@SpdID': 'BW01.NBAY3', '@SpdType': 'T', '@Multiplier': '1', '@Operation': 'PUSH',
         '@Value': '500'},
        {'@TermID': '1', '@SpdType': 'U', '@Multiplier': '2', '@Operation': 'RSD'},
    ]

    assert _rpn_stack(equation) == [1320.0, 500, 550]

    assert _rpn_calc(equation) == 1320


def test_roll_stack_up():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.8.4"""

    equation = [
        {'@TermID': '1', '@SpdID': 'BW01.NBAY1', '@SpdType': 'T', '@Multiplier': '1', '@Value': '660'},
        {'@TermID': '1', '@SpdID': 'BW01.NBAY2', '@SpdType': 'T', '@Multiplier': '1', '@Operation': 'PUSH',
         '@Value': '550'},
        {'@TermID': '1', '@SpdID': 'BW01.NBAY3', '@SpdType': 'T', '@Multiplier': '1', '@Operation': 'PUSH',
         '@Value': '500'},
        {'@TermID': '1', '@SpdType': 'U', '@Multiplier': '2', '@Operation': 'RSU'},
    ]

    assert _rpn_stack(equation) == [1100.0, 660, 500]

    assert _rpn_calc(equation) == 1100


def test_pop():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.8.4"""

    equation = [
        {'@TermID': '1', '@SpdID': 'YWPS1.VYP21', '@SpdType': 'T', '@Multiplier': '1', '@Value': '100'},
        {'@TermID': '1', '@SpdID': 'YWPS2.VYP22', '@SpdType': 'T', '@Multiplier': '-1', '@Operation': 'PUSH',
         '@Value': '350'},
        {'@TermID': '1', '@SpdType': 'U', '@Multiplier': '1', '@Operation': 'POP'},
    ]

    assert _rpn_calc(equation) == 100


def test_exchange_if_less_than_or_equal_to_zero():
    """Test example from AEMO's Constraint Implementation Guidelines June 2015 Appendix A.8.4"""

    equation = [
        {'@TermID': '1', '@SpdID': 'YWPS1.VYP21', '@SpdType': 'T', '@Multiplier': '1', '@Value': '100'},
        {'@TermID': '1', '@SpdID': 'YWPS2.VYP22', '@SpdType': 'T', '@Multiplier': '-1', '@Operation': 'PUSH',
         '@Value': '350'},
        {'@TermID': '1', '@SpdType': 'S', '@Multiplier': '1', '@Operation': 'POP', '@Value': '0'},
        {'@TermID': '1', '@SpdType': 'U', '@Multiplier': '2', '@Operation': 'EXLEZ'},
    ]

    assert _rpn_calc(equation) == 200

================
File: tests\test_solver_interface.py
================
import pandas as pd
from pandas._testing import assert_frame_equal
from nempy.spot_markert_backend import solver_interface


def test_dispatch():
    si = solver_interface.InterfaceToSolver()

    decision_variables = pd.DataFrame({
            'unit': ['A', 'A', 'B', 'B'],
            'upper_bound': [1, 6, 5, 7],
            'variable_id': [4, 5, 6, 7],
            'lower_bound': [0.0, 0.0, 0.0, 0.0],
            'type': ['continuous', 'continuous', 'continuous', 'continuous'],
    })

    si.add_variables(decision_variables)

    constraints_rhs_and_type = pd.DataFrame({
            'constraint_id': [0, 1],
            'type': ['<=', '<='],
            'rhs': [5, 15]
        })

    market_rhs_and_type = pd.DataFrame({
            'constraint_id': [2],
            'type': ['='],
            'rhs': [15]
    })

    rhs_and_type = pd.concat([constraints_rhs_and_type, market_rhs_and_type])

    constraints_lhs_coefficient = pd.DataFrame({
        'constraint_id': [0, 0, 1, 1, 2, 2, 2, 2],
        'variable_id': [4, 5, 6, 7, 4, 5, 6, 7],
        'coefficient': [1, 1, 1, 1, 1, 1, 1, 1]
    })

    si.add_constraints(constraints_lhs_coefficient, rhs_and_type)

    objective_function = pd.DataFrame({
            'variable_id': [4, 5, 6, 7],
            'cost': [0, 1, 2, 3]
    })

    si.add_objective_function(objective_function)

    si.optimize()

    si.linear_mip_model.optimize()

    decision_variables['value'] = si.get_optimal_values_of_decision_variables(decision_variables)

    prices = si.price_constraints([2])

    market_rhs_and_type['price'] = market_rhs_and_type['constraint_id'].map(prices)

    expected_decision_variables = pd.DataFrame({
            'unit': ['A', 'A', 'B', 'B'],
            'upper_bound': [1, 6, 5, 7],
            'variable_id': [4, 5, 6, 7],
            'lower_bound': [0.0, 0.0, 0.0, 0.0],
            'type': ['continuous', 'continuous', 'continuous', 'continuous'],
            'value': [1.0, 4.0, 5.0, 5.0]
    })

    expected_market_rhs_and_type = pd.DataFrame({
            'constraint_id': [2],
            'type': ['='],
            'rhs': [15],
            'price': [3.0]
    })

    assert_frame_equal(decision_variables, expected_decision_variables)
    assert_frame_equal(market_rhs_and_type, expected_market_rhs_and_type)

================
File: tests\test_unit_constraints.py
================
import pandas as pd
from pandas._testing import assert_frame_equal
from nempy.spot_markert_backend import unit_constraints


def test_create_constraints():
    unit_limit = pd.DataFrame({
        'unit': ['A', 'B'],
        'upper': [16.0, 23.0]
    })
    next_constraint_id = 4
    rhs_col = 'upper'
    direction = '<='
    output_rhs, output_variable_map = unit_constraints.create_constraints(unit_limit, next_constraint_id, rhs_col,
                                                                          direction)
    expected_rhs = pd.DataFrame({
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'constraint_id': [4, 5],
        'type': ['<=', '<='],
        'rhs': [16.0, 23.0]
    })
    expected_variable_map = pd.DataFrame({
        'constraint_id': [4, 5],
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'coefficient': [1.0, 1.0]
    })
    assert_frame_equal(output_rhs, expected_rhs)
    assert_frame_equal(output_variable_map, expected_variable_map)


def test_one_unit_create_constraints():
    unit_limit = pd.DataFrame({
        'unit': ['A'],
        'upper': [16.0]
    })
    next_constraint_id = 4
    rhs_col = 'upper'
    direction = '<='
    output_rhs, output_variable_map = unit_constraints.create_constraints(unit_limit, next_constraint_id, rhs_col,
                                                                          direction)
    expected_rhs = pd.DataFrame({
        'unit': ['A'],
        'service': ['energy'],
        'constraint_id': [4],
        'type': ['<='],
        'rhs': [16.0],

    })
    expected_variable_map = pd.DataFrame({
        'constraint_id': [4],
        'unit': ['A'],
        'service': ['energy'],
        'coefficient': [1.0],
    })
    assert_frame_equal(output_rhs, expected_rhs)
    assert_frame_equal(output_variable_map, expected_variable_map)


def test_ramp_down():
    unit_limit = pd.DataFrame({
        'unit': ['A', 'B'],
        'initial_output': [16.0, 23.0],
        'ramp_down_rate': [12.0, 36.0]
    })
    next_constraint_id = 4
    dispatch_interval = 5
    output_rhs, output_variable_map = unit_constraints.ramp_down(unit_limit, next_constraint_id, dispatch_interval)
    expected_rhs = pd.DataFrame({
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'constraint_id': [4, 5],
        'type': ['>=', '>='],
        'rhs': [15.0, 20.0]
    })
    expected_variable_map = pd.DataFrame({
        'constraint_id': [4, 5],
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'coefficient': [1.0, 1.0],
    })
    assert_frame_equal(output_rhs.reset_index(drop=True), expected_rhs)
    assert_frame_equal(output_variable_map.reset_index(drop=True), expected_variable_map)


def test_ramp_up():
    unit_limit = pd.DataFrame({
        'unit': ['A', 'B'],
        'initial_output': [16, 23],
        'ramp_up_rate': [12, 36]
    })
    next_constraint_id = 4
    dispatch_interval = 5
    output_rhs, output_variable_map = unit_constraints.ramp_up(unit_limit, next_constraint_id, dispatch_interval)
    expected_rhs = pd.DataFrame({
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'constraint_id': [4, 5],
        'type': ['<=', '<='],
        'rhs': [17.0, 26.0],
    })
    expected_variable_map = pd.DataFrame({
        'constraint_id': [4, 5],
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'coefficient': [1.0, 1.0]
    })
    assert_frame_equal(output_rhs.reset_index(drop=True), expected_rhs)
    assert_frame_equal(output_variable_map.reset_index(drop=True), expected_variable_map)


def test_capacity():
    unit_limit = pd.DataFrame({
        'unit': ['A', 'B'],
        'capacity': [16.0, 23.0]
    })
    next_constraint_id = 4
    output_rhs, output_variable_map = unit_constraints.capacity(unit_limit, next_constraint_id)
    expected_rhs = pd.DataFrame({
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'constraint_id': [4, 5],
        'type': ['<=', '<='],
        'rhs': [16.0, 23.0],
    })
    expected_variable_map = pd.DataFrame({
        'constraint_id': [4, 5],
        'unit': ['A', 'B'],
        'service': ['energy', 'energy'],
        'coefficient': [1.0, 1.0],
    })
    assert_frame_equal(output_rhs.reset_index(drop=True), expected_rhs)
    assert_frame_equal(output_variable_map.reset_index(drop=True), expected_variable_map)

================
File: tests\test_variable_ids.py
================
import pandas as pd
from pandas._testing import assert_frame_equal
from nempy.spot_markert_backend import variable_ids


def test_energy_one_unit():
    bids = pd.DataFrame({
        'unit': ['A'],
        '1': [1.0]
    })
    unit_info = pd.DataFrame({
        'unit': ['A'],
        'region': ['X'],
        'dispatch_type': ['generator']
    })
    next_constraint_id = 4
    output_vars, unit_level_constraint_map, region_level_constraint_map = \
        variable_ids.bids(bids, unit_info, next_constraint_id)
    expected_vars = pd.DataFrame({
        'unit': ['A'],
        'capacity_band': ['1'],
        'service': ['energy'],
        'variable_id': [4],
        'lower_bound': [0.0],
        'upper_bound': [1.0],
        'type': ['continuous']
    })
    expected_unit_constraint_map = pd.DataFrame({
        'variable_id': [4],
        'unit': ['A'],
        'service': ['energy'],
        'coefficient': [1.0]
    })
    expected_regional_constraint_map = pd.DataFrame({
        'variable_id': [4],
        'region': ['X'],
        'service': ['energy'],
        'coefficient': [1.0]
    })
    assert_frame_equal(output_vars, expected_vars)
    assert_frame_equal(unit_level_constraint_map, expected_unit_constraint_map)
    assert_frame_equal(region_level_constraint_map, expected_regional_constraint_map)


def test_energy_two_units():
    bids = pd.DataFrame({
        'unit': ['A', 'B'],
        '1': [1.0, 5.0],
        '2': [6.0, 7.0]
    })
    unit_info = pd.DataFrame({
        'unit': ['A', 'B'],
        'region': ['X', 'Y'],
        'dispatch_type': ['generator', 'load']
    })
    next_constraint_id = 4
    output_vars, unit_level_constraint_map, region_level_constraint_map = \
        variable_ids.bids(bids, unit_info, next_constraint_id)
    expected_vars = pd.DataFrame({
        'unit': ['A', 'A', 'B', 'B'],
        'capacity_band': ['1', '2', '1', '2'],
        'service': ['energy', 'energy', 'energy', 'energy'],
        'variable_id': [4, 5, 6, 7],
        'lower_bound': [0.0, 0.0, 0.0, 0.0],
        'upper_bound': [1.0, 6.0, 5.0, 7.0],
        'type': ['continuous', 'continuous', 'continuous', 'continuous']
    })
    expected_unit_constraint_map = pd.DataFrame({
        'variable_id': [4, 5, 6, 7],
        'unit': ['A', 'A', 'B', 'B'],
        'service': ['energy', 'energy', 'energy', 'energy'],
        'coefficient': [1.0, 1.0, 1.0, 1.0]
    })
    expected_region_constraint_map = pd.DataFrame({
        'variable_id': [4, 5, 6, 7],
        'region': ['X', 'X', 'Y', 'Y'],
        'service': ['energy', 'energy', 'energy', 'energy'],
        'coefficient': [1.0, 1.0, -1.0, -1.0]
    })
    assert_frame_equal(output_vars, expected_vars)
    assert_frame_equal(unit_level_constraint_map, expected_unit_constraint_map)
    assert_frame_equal(region_level_constraint_map, expected_region_constraint_map)