Using local postgres for pytest (#843)

1. Spinning local postgres container for testing
(#836)
2. Using conftest for test fixtures to avoid importing fixture
dependencies in tests
3. Fixing rounding bugs with fixedpoint
(delvtech/fixedpointmath#21)

README.md

@@ -67,14 +67,21 @@ Due to a lack of known precision, operations against Python floats are not allow
 However, operations against `int` are allowed.
 In this case, the `int` _argument_ is assumed to be "unscaled", i.e. if you write `int(8) * FixedPoint(8)` we will scale up the first variable return a `FixedPoint` number that represents the float `64.0` in 18-decimal FixedPoint format.
 So in this example the internal representation of that operation would be `64*10**18`.
-If you cast FixedPoint numbers to ints or floats you will get "unscaled" representation, e.g. `float(FixedPoint(8.0)) == 8.0` and `int(FixedPoint(8.528)) == 8`.
+If you cast FixedPoint numbers to ints or floats you will get "unscaled" representation, e.g. `float(FixedPoint("8.0")) == 8.0` and `int(FixedPoint("8.528")) == 8`.
 
-If you want the integer scaled representation, which can be useful for communicating with Solidity contracts, you must ask for it explicitly, e.g. `FixedPoint(8.52).scaled_value == 8520000000000000000`.
+If you want the integer scaled representation, which can be useful for communicating with Solidity contracts, you must ask for it explicitly, e.g. `FixedPoint("8.52").scaled_value == 8520000000000000000`.
 Conversely, if you want to initialize a FixedPoint variable using the scaled integer representation, then you need to instantiate the variable using the `scaled_value` argument, e.g. `FixedPoint(scaled_value=8)`.
 In that example, the internal representation is `8`, so casting it to a float would produce a small value: `float(FixedPoint(scaled_value=8)) == 8e-18`.
 
 To understand more, we recommend that you study the fixed point tests and source implementation in `elfpy/math/`.
 
+Warning! Using floating point as a constructor to FixedPoint can cause loss of precision. For example, 
+```
+>>> FixedPoint(1e18)
+FixedPoint("1000000000000000042.420637374017961984")
+```
+Allowing floating point in the constructor of FixedPoint will be removed in a future version of `fixedpointmath`.
+
 ## Modifying configuration for agent deployment
 
 Follow `lib/agent0/README.md` for agent deployment.

conftest.py

@@ -1,7 +1,24 @@
+# Ignore docstrings for this file
+# pylint: disable=missing-docstring
+
+
+import os
+
+import pytest
+from agent0.test_fixtures import cycle_trade_policy
+from chainsync.test_fixtures import database_engine, db_session, dummy_session, psql_docker
+from ethpy.test_fixtures import local_chain, local_hyperdrive_chain
+
 # Hack to allow for vscode debugger to throw exception immediately
 # instead of allowing pytest to catch the exception and report
 # Based on https://stackoverflow.com/questions/62419998/how-can-i-get-pytest-to-not-catch-exceptions/62563106#62563106
 
+# IMPORTANT NOTE!!!!!
+# If you end up using this debugging method, this will catch exceptions before teardown of fixtures
+# This means that the local postgres fixture (which launches a docker container) will not automatically
+# be cleaned up if you, e.g., use the debugger and a db test fails. Make sure to manually clean up.
+# TODO maybe automatically close the container on catch here
+
 # Use this in conjunction with the following launch.json configuration:
 #      {
 #        "name": "Debug Current Test",
@@ -15,15 +32,6 @@
 #            "_PYTEST_RAISE": "1"
 #        },
 #      },
-
-# Ignore docstrings for this file
-# pylint: disable=missing-docstring
-
-
-import os
-
-import pytest
-
 if os.getenv("_PYTEST_RAISE", "0") != "0":
 
     @pytest.hookimpl(tryfirst=True)
@@ -33,3 +41,17 @@ def pytest_exception_interact(call):
     @pytest.hookimpl(tryfirst=True)
     def pytest_internalerror(excinfo):
         raise excinfo.value
+
+
+# Importing all fixtures here and defining here
+# This allows for users of fixtures to not have to import all dependency fixtures when running
+# TODO this means pytest can only be ran from this directory
+__all__ = [
+    "database_engine",
+    "db_session",
+    "dummy_session",
+    "psql_docker",
+    "local_chain",
+    "local_hyperdrive_chain",
+    "cycle_trade_policy",
+]

lib/agent0/agent0/base/config/agent_config.py

@@ -39,7 +39,7 @@ class AgentConfig:
     name: str = "BoringBotty"
     base_budget_wei: Budget | int = Budget()
     eth_budget_wei: Budget | int = Budget(min_wei=0, max_wei=0)
-    slippage_tolerance: FixedPoint = FixedPoint(0.0001)  # default to 0.01%
+    slippage_tolerance: FixedPoint = FixedPoint("0.0001")  # default to 0.01%
     number_of_agents: int = 1
     private_keys: list[str] | None = None
     init_kwargs: dict = field(default_factory=dict)

lib/agent0/agent0/base/config/budget.py

@@ -16,10 +16,10 @@ class Budget:
     Wei in the variables below refers to the smallest unit of base, not to ETH.
     """
 
-    mean_wei: int = int(5_000 * 1e18)
-    std_wei: int = int(2_000 * 1e18)
-    min_wei: int = int(1_000 * 1e18)
-    max_wei: int = int(10_000 * 1e18)
+    mean_wei: int = FixedPoint(5_000).scaled_value
+    std_wei: int = FixedPoint(2_000).scaled_value
+    min_wei: int = FixedPoint(1_000).scaled_value
+    max_wei: int = FixedPoint(10_000).scaled_value
 
     def sample_budget(self, rng: NumpyGenerator) -> FixedPoint:
         """Return a sample from a clipped normal distribution.

lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py

@@ -229,7 +229,7 @@ async def async_match_contract_call_to_trade(
 
     # TODO: The following variables are hard coded for now, but should be specified in the trade spec
     min_apr = int(1)
-    max_apr = int(1e18)
+    max_apr = FixedPoint(1).scaled_value
     as_underlying = True
     match trade.action_type:
         case HyperdriveActionType.INITIALIZE_MARKET:

lib/agent0/agent0/test_fixtures/cycle_trade_policy.py

@@ -44,7 +44,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis
                     market_type=MarketType.HYPERDRIVE,
                     market_action=HyperdriveMarketAction(
                         action_type=HyperdriveActionType.ADD_LIQUIDITY,
-                        trade_amount=FixedPoint(scaled_value=int(11111e18)),
+                        trade_amount=FixedPoint(11111),
                         wallet=wallet,
                     ),
                 )
@@ -56,7 +56,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis
                     market_type=MarketType.HYPERDRIVE,
                     market_action=HyperdriveMarketAction(
                         action_type=HyperdriveActionType.OPEN_LONG,
-                        trade_amount=FixedPoint(scaled_value=int(22222e18)),
+                        trade_amount=FixedPoint(22222),
                         wallet=wallet,
                     ),
                 )
@@ -68,7 +68,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis
                     market_type=MarketType.HYPERDRIVE,
                     market_action=HyperdriveMarketAction(
                         action_type=HyperdriveActionType.OPEN_SHORT,
-                        trade_amount=FixedPoint(scaled_value=int(33333e18)),
+                        trade_amount=FixedPoint(33333),
                         wallet=wallet,
                     ),
                 )
@@ -138,7 +138,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis
                     market_type=MarketType.HYPERDRIVE,
                     market_action=HyperdriveMarketAction(
                         action_type=HyperdriveActionType.OPEN_LONG,
-                        trade_amount=FixedPoint(scaled_value=int(1e18)),
+                        trade_amount=FixedPoint(1),
                         wallet=wallet,
                     ),
                 )

lib/agent0/bin/checkpoint_bot.py

@@ -20,6 +20,7 @@
     smart_contract_transact,
 )
 from ethpy.hyperdrive import fetch_hyperdrive_address_from_url, get_hyperdrive_config
+from fixedpointmath import FixedPoint
 from web3.contract.contract import Contract
 
 # The portion of the checkpoint that the bot will wait before attempting to
@@ -69,7 +70,7 @@ def main() -> None:
     )
 
     # Fund the checkpoint sender with some ETH.
-    balance = int(100e18)
+    balance = FixedPoint(100).scaled_value
     sender = EthAgent(Account().create("CHECKPOINT_BOT"))
     set_anvil_account_balance(web3, sender.address, balance)
     logging.info("Successfully funded the sender=%s.", sender.address)

lib/agent0/examples/example_agent.py

@@ -37,7 +37,7 @@ def __init__(
         rng: NumpyGenerator | None = None,
         slippage_tolerance: FixedPoint | None = None,
         # Add additional parameters for custom policy here
-        static_trade_amount_wei: int = int(100e18),  # 100 base
+        static_trade_amount_wei: int = FixedPoint(100).scaled_value,  # 100 base
     ):
         self.static_trade_amount_wei = static_trade_amount_wei
         # We want to do a sequence of trades one at a time, so we keep an internal counter based on
@@ -174,10 +174,10 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis
     AgentConfig(
         policy=CycleTradesPolicy,
         number_of_agents=1,
-        slippage_tolerance=FixedPoint(0.0001),
-        base_budget_wei=int(10_000e18),  # 10k base
-        eth_budget_wei=int(10e18),  # 10 base
-        init_kwargs={"static_trade_amount_wei": int(100e18)},  # 100 base static trades
+        slippage_tolerance=FixedPoint("0.0001"),
+        base_budget_wei=FixedPoint(10_000).scaled_value,  # 10k base
+        eth_budget_wei=FixedPoint(10).scaled_value,  # 10 base
+        init_kwargs={"static_trade_amount_wei": FixedPoint(100).scaled_value},  # 100 base static trades
     ),
 ]
 

lib/agent0/examples/hyperdrive_agents.py

@@ -32,35 +32,35 @@
     AgentConfig(
         policy=Policies.random_agent,
         number_of_agents=3,
-        slippage_tolerance=FixedPoint(0.0001),
+        slippage_tolerance=FixedPoint("0.0001"),
         base_budget_wei=Budget(
-            mean_wei=int(5_000e18),  # 5k base
-            std_wei=int(1_000e18),  # 1k base
+            mean_wei=FixedPoint(5_000).scaled_value,  # 5k base
+            std_wei=FixedPoint(1_000).scaled_value,  # 1k base
             min_wei=1,  # 1 WEI base
-            max_wei=int(100_000e18),  # 100k base
+            max_wei=FixedPoint(100_000).scaled_value,  # 100k base
         ),
-        eth_budget_wei=Budget(min_wei=int(1e18), max_wei=int(1e18)),
-        init_kwargs={"trade_chance": FixedPoint(0.8)},
+        eth_budget_wei=Budget(min_wei=FixedPoint(1).scaled_value, max_wei=FixedPoint(1).scaled_value),
+        init_kwargs={"trade_chance": FixedPoint("0.8")},
     ),
     AgentConfig(
         policy=Policies.long_louie,
         number_of_agents=0,
         # Fixed budgets
-        base_budget_wei=int(5_000e18),  # 5k base
-        eth_budget_wei=int(1e18),  # 1 base
-        init_kwargs={"trade_chance": FixedPoint(0.8), "risk_threshold": FixedPoint(0.9)},
+        base_budget_wei=FixedPoint(5_000).scaled_value,  # 5k base
+        eth_budget_wei=FixedPoint(1).scaled_value,  # 1 base
+        init_kwargs={"trade_chance": FixedPoint("0.8"), "risk_threshold": FixedPoint("0.9")},
     ),
     AgentConfig(
         policy=Policies.short_sally,
         number_of_agents=0,
         base_budget_wei=Budget(
-            mean_wei=int(5_000e18),  # 5k base
-            std_wei=int(1_000e18),  # 1k base
+            mean_wei=FixedPoint(5_000).scaled_value,  # 5k base
+            std_wei=FixedPoint(1_000).scaled_value,  # 1k base
             min_wei=1,  # 1 WEI base
-            max_wei=int(100_000e18),  # 100k base
+            max_wei=FixedPoint(100_000).scaled_value,  # 100k base
         ),
-        eth_budget_wei=Budget(min_wei=int(1e18), max_wei=int(1e18)),
-        init_kwargs={"trade_chance": FixedPoint(0.8), "risk_threshold": FixedPoint(0.8)},
+        eth_budget_wei=Budget(min_wei=FixedPoint(1).scaled_value, max_wei=FixedPoint(1).scaled_value),
+        init_kwargs={"trade_chance": FixedPoint("0.8"), "risk_threshold": FixedPoint("0.8")},
     ),
 ]
 

lib/chainsync/chainsync/db/base/interface.py

@@ -8,7 +8,7 @@
 
 import pandas as pd
 import sqlalchemy
-from chainsync import build_postgres_config
+from chainsync import PostgresConfig, build_postgres_config
 from sqlalchemy import URL, Column, Engine, MetaData, String, Table, create_engine, exc, func, inspect
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.ext.declarative import declared_attr
@@ -56,15 +56,16 @@ def drop_table(session: Session, table_name: str) -> None:
     table.drop(checkfirst=True, bind=bind)
 
 
-def initialize_engine() -> Engine:
+def initialize_engine(postgres_config: PostgresConfig | None = None) -> Engine:
     """Initializes the postgres engine from config
 
     Returns
     -------
     Engine
         The initialized engine object connected to postgres
     """
-    postgres_config = build_postgres_config()
+    if postgres_config is None:
+        postgres_config = build_postgres_config()
 
     url_object = URL.create(
         drivername="postgresql",

lib/chainsync/chainsync/db/base/schema_test.py

@@ -13,9 +13,6 @@ class TestUserMapTable:
 
     def test_create_user_map(self, db_session):
         """Create and entry"""
-        # Note: this test is using in-memory sqlite, which doesn't seem to support
-        # autoincrement ids without init, whereas postgres does this with no issues
-        # Hence, we explicitly add id here
         user_map = UserMap(address="1", username="a")
         db_session.add(user_map)
         db_session.commit()

lib/chainsync/chainsync/db/hyperdrive/interface_test.py

@@ -6,9 +6,6 @@
 import pytest
 from chainsync.db.base import get_latest_block_number_from_table
 
-# Ignoring unused import warning, fixtures are used through variable name
-from chainsync.test_fixtures import db_session  # pylint: disable=unused-import
-
 from .interface import (
     add_checkpoint_infos,
     add_pool_config,
@@ -29,10 +26,8 @@
 )
 from .schema import CheckpointInfo, HyperdriveTransaction, PoolConfig, PoolInfo, WalletDelta, WalletInfo
 
-# fixture arguments in test function have to be the same as the fixture name
-# pylint: disable=redefined-outer-name
-
 
+# These tests are using fixtures defined in conftest.py
 class TestTransactionInterface:
     """Testing postgres interface for transaction table"""
 
@@ -151,10 +146,6 @@ def test_get_pool_config(self, db_session):
 
         pool_config_df_1 = get_pool_config(db_session)
         assert len(pool_config_df_1) == 1
-        # TODO In testing, we use sqlite, which does not implement the fixed point Numeric type
-        # Internally, they store Numeric types as floats, hence we see rounding errors in testing
-        # This does not happen in postgres, where these values match exactly.
-        # https://github.com/delvtech/elf-simulations/issues/836
         np.testing.assert_array_equal(pool_config_df_1["initialSharePrice"], np.array([3.2]))
 
         pool_config_2 = PoolConfig(contractAddress="1", initialSharePrice=Decimal("3.4"))
@@ -185,9 +176,7 @@ def test_pool_config_verify(self, db_session):
         assert pool_config_df_1.loc[0, "initialSharePrice"] == 3.2
 
         # Nothing should happen if we give the same pool_config
-        # TODO Below is a hack due to sqlite not having numerics
-        # We explicitly print 18 spots after floating point to match rounding error in sqlite
-        pool_config_2 = PoolConfig(contractAddress="0", initialSharePrice=Decimal(f"{3.2:.18f}"))
+        pool_config_2 = PoolConfig(contractAddress="0", initialSharePrice=Decimal("3.2"))
         add_pool_config(pool_config_2, db_session)
         pool_config_df_2 = get_pool_config(db_session)
         assert len(pool_config_df_2) == 1

lib/chainsync/chainsync/db/hyperdrive/schema.py

@@ -5,7 +5,7 @@
 from typing import Union
 
 from chainsync.db.base import Base
-from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Integer, Numeric, String
+from sqlalchemy import BigInteger, Boolean, DateTime, Integer, Numeric, String
 from sqlalchemy.orm import Mapped, mapped_column
 
 # pylint: disable=invalid-name
@@ -35,7 +35,7 @@ class PoolConfig(Base):
     curveFee: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None)
     flatFee: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None)
     governanceFee: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None)
-    oracleSize: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None)
+    oracleSize: Mapped[Union[int, None]] = mapped_column(Integer, default=None)
     updateGap: Mapped[Union[int, None]] = mapped_column(Integer, default=None)
     invTimeStretch: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None)
     updateGap: Mapped[Union[int, None]] = mapped_column(Integer, default=None)
@@ -85,13 +85,9 @@ class WalletInfo(Base):
     __tablename__ = "walletinfo"
 
     # Default table primary key
-    # Note that we use postgres in production and sqlite in testing, but sqlite has issues with
-    # autoincrement with BigIntegers. Hence, we use the Integer variant when using sqlite in tests
-    id: Mapped[int] = mapped_column(
-        BigInteger().with_variant(Integer, "sqlite"), primary_key=True, init=False, autoincrement=True
-    )
+    id: Mapped[int] = mapped_column(BigInteger(), primary_key=True, init=False, autoincrement=True)
 
-    blockNumber: Mapped[int] = mapped_column(BigInteger, ForeignKey("poolinfo.blockNumber"), index=True)
+    blockNumber: Mapped[int] = mapped_column(BigInteger, index=True)
     walletAddress: Mapped[Union[str, None]] = mapped_column(String, index=True, default=None)
     # baseTokenType can be BASE, LONG, SHORT, LP, or WITHDRAWAL_SHARE
     baseTokenType: Mapped[Union[str, None]] = mapped_column(String, index=True, default=None)
@@ -111,13 +107,9 @@ class WalletDelta(Base):
     __tablename__ = "walletdelta"
 
     # Default table primary key
-    # Note that we use postgres in production and sqlite in testing, but sqlite has issues with
-    # autoincrement with BigIntegers. Hence, we use the Integer variant when using sqlite in tests
-    id: Mapped[int] = mapped_column(
-        BigInteger().with_variant(Integer, "sqlite"), primary_key=True, init=False, autoincrement=True
-    )
-    transactionHash: Mapped[str] = mapped_column(String, ForeignKey("transactions.transactionHash"), index=True)
-    blockNumber: Mapped[int] = mapped_column(BigInteger, ForeignKey("poolinfo.blockNumber"), index=True)
+    id: Mapped[int] = mapped_column(BigInteger(), primary_key=True, init=False, autoincrement=True)
+    transactionHash: Mapped[str] = mapped_column(String, index=True)
+    blockNumber: Mapped[int] = mapped_column(BigInteger, index=True)
     walletAddress: Mapped[Union[str, None]] = mapped_column(String, index=True, default=None)
     # baseTokenType can be BASE, LONG, SHORT, LP, or WITHDRAWAL_SHARE
     baseTokenType: Mapped[Union[str, None]] = mapped_column(String, index=True, default=None)
@@ -138,15 +130,11 @@ class HyperdriveTransaction(Base):
     __tablename__ = "transactions"
 
     # Default table primary key
-    # Note that we use postgres in production and sqlite in testing, but sqlite has issues with
-    # autoincrement with BigIntegers. Hence, we use the Integer variant when using sqlite in tests
-    id: Mapped[int] = mapped_column(
-        BigInteger().with_variant(Integer, "sqlite"), primary_key=True, init=False, autoincrement=True
-    )
+    id: Mapped[int] = mapped_column(BigInteger(), primary_key=True, init=False, autoincrement=True)
     transactionHash: Mapped[str] = mapped_column(String, index=True, unique=True)
 
     #### Fields from base transactions ####
-    blockNumber: Mapped[int] = mapped_column(BigInteger, ForeignKey("poolinfo.blockNumber"), index=True)
+    blockNumber: Mapped[int] = mapped_column(BigInteger, index=True)
     transactionIndex: Mapped[Union[int, None]] = mapped_column(Integer, default=None)
     nonce: Mapped[Union[int, None]] = mapped_column(Integer, default=None)
     # Transaction receipt to/from