From be7fafbab031b52ba63f73fd56fd0bf6065790c7 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 01/59] feat: some treasury stuff --- yearn/treasury/accountant/other_expenses/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yearn/treasury/accountant/other_expenses/general.py b/yearn/treasury/accountant/other_expenses/general.py index 6fa3fe953..1eb7f30e3 100644 --- a/yearn/treasury/accountant/other_expenses/general.py +++ b/yearn/treasury/accountant/other_expenses/general.py @@ -113,4 +113,4 @@ def is_yfi_dot_eth(tx: TreasuryTx) -> bool: def is_yyper_contest(tx: TreasuryTx) -> bool: """Grant for a vyper compiler audit context, vyper-context.eth""" - return tx in HashMatcher([["0xb8bb3728fdfb49d7c86c08dba8e3586e3761f13d2c88fa6fab80227b6a3f4519", Filter('log_index', 202)]]) \ No newline at end of file + return tx in HashMatcher([["0xb8bb3728fdfb49d7c86c08dba8e3586e3761f13d2c88fa6fab80227b6a3f4519", Filter('log_index', 202)]]) From 434def3ed32e166a8386933180748fafac1f98a2 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 02/59] feat: delay curve loading --- yearn/apy/curve/simple.py | 5 +++-- yearn/constants.py | 10 +++++++++- yearn/prices/curve.py | 18 ++++++++---------- yearn/prices/magic.py | 6 ++++-- yearn/special.py | 5 +++-- yearn/v1/vaults.py | 13 ++++++------- yearn/v2/vaults.py | 9 ++++----- 7 files changed, 37 insertions(+), 29 deletions(-) diff --git a/yearn/apy/curve/simple.py b/yearn/apy/curve/simple.py index b54042943..c8ce23d0c 100644 --- a/yearn/apy/curve/simple.py +++ b/yearn/apy/curve/simple.py @@ -21,6 +21,7 @@ from y.time import get_block_timestamp_async from y.utils.dank_mids import dank_w3 +from yearn import constants from yearn.apy.common import (SECONDS_PER_WEEK, SECONDS_PER_YEAR, Apy, ApyError, ApyFees, ApySamples, SharePricePoint, calculate_roi) @@ -164,7 +165,7 @@ async def calculate_simple(vault, gauge: Gauge, samples: ApySamples) -> Apy: raise ValueError(f"Error! Could not find price for {gauge.lp_token} at block {block}") crv_price, pool_price = await asyncio.gather( - magic.get_price(curve.crv, block=block, sync=False), + magic.get_price(constants.CRV, block=block, sync=False), gauge.pool.get_virtual_price.coroutine(block_identifier=block) ) gauge_weight = gauge.gauge_weight @@ -461,7 +462,7 @@ async def get_detailed_apy_data(self, base_asset_price, pool_price, base_apr) -> async def _get_cvx_emissions_converted_to_crv(self) -> float: """The amount of CVX emissions at the current block for a given pool, converted to CRV (from a pricing standpoint) to ease calculation of total APY.""" crv_price, cvx = await asyncio.gather( - magic.get_price(curve.crv, block=self.block, sync=False), + magic.get_price(constants.CRV, block=self.block, sync=False), Contract.coroutine(addresses[chain.id]['cvx']), ) total_cliff = 1e3 # the total number of cliffs to happen diff --git a/yearn/constants.py b/yearn/constants.py index 467c8dd7d..835e10f0f 100644 --- a/yearn/constants.py +++ b/yearn/constants.py @@ -112,4 +112,12 @@ TREASURY_WALLETS = {convert.to_address(address) for address in TREASURY_WALLETS} -RANDOMIZE_EXPORTS = bool(os.environ.get("RANDOMIZE_EXPORTS")) \ No newline at end of file +RANDOMIZE_EXPORTS = bool(os.environ.get("RANDOMIZE_EXPORTS")) + +CRV = { + Network.Mainnet: "0xD533a949740bb3306d119CC777fa900bA034cd52", + Network.Gnosis: "0x712b3d230f3c1c19db860d80619288b1f0bdd0bd", + Network.Fantom: "0x1E4F97b9f9F913c46F1632781732927B9019C68b", + Network.Arbitrum: "0x11cDb42B0EB46D95f990BeDD4695A6e3fA034978", + Network.Optimism: "0x0994206dfE8De6Ec6920FF4D779B0d950605Fb53", +}.get(chain.id, None) diff --git a/yearn/prices/curve.py b/yearn/prices/curve.py index aa089bbd2..37985a084 100644 --- a/yearn/prices/curve.py +++ b/yearn/prices/curve.py @@ -29,6 +29,7 @@ from y.networks import Network from y.prices import magic +from yearn import constants from yearn.decorators import sentry_catch_all, wait_or_exit_after from yearn.events import decode_logs, get_logs_asap from yearn.exceptions import UnsupportedNetwork @@ -64,7 +65,6 @@ curve_contracts = { Network.Mainnet: { 'address_provider': ADDRESS_PROVIDER, - 'crv': '0xD533a949740bb3306d119CC777fa900bA034cd52', 'voting_escrow': '0x5f3b5DfEb7B28CDbD7FAba78963EE202a494e2A2', 'gauge_controller': '0x2F50D538606Fa9EDD2B11E2446BEb18C9D5846bB', }, @@ -72,19 +72,15 @@ # Curve has not properly initialized the provider. contract(self.address_provider.get_address(5)) returns 0x0. # CurveRegistry class has extra handling to fetch registry in this case. 'address_provider': ADDRESS_PROVIDER, - 'crv': '0x712b3d230f3c1c19db860d80619288b1f0bdd0bd', }, Network.Fantom: { 'address_provider': ADDRESS_PROVIDER, - 'crv': '0x1E4F97b9f9F913c46F1632781732927B9019C68b', }, Network.Arbitrum: { 'address_provider': ADDRESS_PROVIDER, - 'crv': '0x11cDb42B0EB46D95f990BeDD4695A6e3fA034978', }, Network.Optimism: { 'address_provider': ADDRESS_PROVIDER, - 'crv': '0x0994206dfE8De6Ec6920FF4D779B0d950605Fb53', } } @@ -104,8 +100,6 @@ class Ids(IntEnum): Curve_Tricrypto_Factory = 11 class CurveRegistry(metaclass=Singleton): - - @wait_or_exit_after def __init__(self) -> None: if chain.id not in curve_contracts: raise UnsupportedNetwork("curve is not supported on this network") @@ -115,7 +109,6 @@ def __init__(self) -> None: self.voting_escrow = contract(addrs['voting_escrow']) self.gauge_controller = contract(addrs['gauge_controller']) - self.crv = contract(addrs['crv']) self.identifiers = defaultdict(list) # id -> versions self.registries = defaultdict(set) # registry -> pools self.factories = defaultdict(set) # factory -> pools @@ -126,7 +119,11 @@ def __init__(self) -> None: self._done = threading.Event() self._thread = threading.Thread(target=self.watch_events, daemon=True) self._has_exception = False - self._thread.start() + + @wait_or_exit_after + def ensure_loaded(self): + if not self._thread._started.is_set(): + self._thread.start() @sentry_catch_all def watch_events(self) -> None: @@ -270,6 +267,7 @@ def get_pool(self, token: AddressOrContract) -> EthAddress: """ Get Curve pool (swap) address by LP token address. Supports factory pools. """ + self.ensure_loaded() token = to_address(token) if token in self.token_to_pool: return self.token_to_pool[token] @@ -564,7 +562,7 @@ async def calculate_apy(self, gauge: Contract, lp_token: AddressOrContract, bloc block=block, ) crv_price, token_price, results = await asyncio.gather( - magic.get_price(self.crv, block=block, sync=False), + magic.get_price(constants.CRV, block=block, sync=False), magic.get_price(lp_token, block=block, sync=False), results ) diff --git a/yearn/prices/magic.py b/yearn/prices/magic.py index 261a5ec29..9fb0503d2 100644 --- a/yearn/prices/magic.py +++ b/yearn/prices/magic.py @@ -8,6 +8,7 @@ from y.exceptions import PriceError from y.networks import Network +from yearn.constants import CRV from yearn.prices import constants, curve from yearn.prices.aave import aave from yearn.prices.balancer import balancer as bal @@ -30,6 +31,7 @@ async def _get_price(token: AnyAddressType, block: Optional[Block]) -> float: if chain.id == Network.Mainnet: # fixes circular import from yearn.special import Backscratcher + # no liquid market for yveCRV-DAO -> return CRV token price if token == Backscratcher().vault.address and block < 11786563: return await _get_price("0xD533a949740bb3306d119CC777fa900bA034cd52", block) @@ -121,8 +123,8 @@ def find_price( elif chain.id == Network.Mainnet: # no liquid market for yveCRV-DAO -> return CRV token price if token == Backscratcher().vault.address and block < 11786563: - if curve.curve and curve.curve.crv: - return get_price(curve.curve.crv, block=block) + if curve.curve and CRV: + return get_price(CRV, block=block) # no liquidity for curve pool (yvecrv-f) -> return 0 elif token == "0x7E46fd8a30869aa9ed55af031067Df666EfE87da" and block < 14987514: return 0 diff --git a/yearn/special.py b/yearn/special.py index 609fa9475..e77275caf 100644 --- a/yearn/special.py +++ b/yearn/special.py @@ -10,9 +10,11 @@ from y.contracts import contract_creation_block_async from y.exceptions import PriceError, yPriceMagicError +from yearn import constants from yearn.apy.common import (Apy, ApyBlocks, ApyError, ApyFees, ApyPoints, ApySamples) from yearn.common import Tvl +from yearn.prices.curve import curve from yearn.utils import Singleton if TYPE_CHECKING: @@ -69,10 +71,9 @@ def __init__(self): self.proxy = Contract("0xF147b8125d2ef93FB6965Db97D6746952a133934") async def _locked(self, block=None) -> Tuple[float,float]: - from yearn.prices.curve import curve crv_locked, crv_price = await asyncio.gather( curve.voting_escrow.balanceOf["address"].coroutine(self.proxy, block_identifier=block), - magic.get_price(curve.crv, block=block, sync=False), + magic.get_price(constants.CRV, block=block, sync=False), ) crv_locked /= 1e18 return crv_locked, crv_price diff --git a/yearn/v1/vaults.py b/yearn/v1/vaults.py index dcdd922e4..c51b688fb 100644 --- a/yearn/v1/vaults.py +++ b/yearn/v1/vaults.py @@ -1,9 +1,9 @@ import asyncio import logging from dataclasses import dataclass -from functools import cached_property from typing import TYPE_CHECKING, Optional +from async_property import async_cached_property from brownie import ZERO_ADDRESS, interface from brownie.network.contract import InterfaceContainer from dank_mids.brownie_patch import patch_contract @@ -14,6 +14,7 @@ from yearn import constants from yearn.common import Tvl from yearn.multicall2 import fetch_multicall_async +from yearn.prices.curve import curve from yearn.utils import contract from yearn.v1 import constants @@ -66,10 +67,9 @@ async def get_controller(self, block=None): return self.controller return contract(self.vault.controller(block_identifier=block)) - @cached_property - def is_curve_vault(self): - from yearn.prices.curve import curve - return curve.get_pool(str(self.token)) is not None + @async_cached_property + async def is_curve_vault(self): + return await magic.curve.get_pool(str(self.token)) is not None async def describe(self, block=None): info = {} @@ -94,7 +94,7 @@ async def describe(self, block=None): attrs["max"] = [self.vault, "max"] # new curve voter proxy vaults - if self.is_curve_vault and hasattr(strategy, "proxy"): + if await self.is_curve_vault and hasattr(strategy, "proxy"): vote_proxy, gauge = await fetch_multicall_async( [strategy, "voter"], # voter is static, can pin [strategy, "gauge"], # gauge is static per strategy, can cache @@ -103,7 +103,6 @@ async def describe(self, block=None): # guard historical queries where there are no vote_proxy and gauge # for block <= 10635293 (2020-08-11) if vote_proxy and gauge: - from yearn.prices.curve import curve vote_proxy = patch_contract(interface.CurveYCRVVoter(vote_proxy), dank_w3) gauge = contract(gauge) boost, _apy = await asyncio.gather( diff --git a/yearn/v2/vaults.py b/yearn/v2/vaults.py index b229eac90..b8e1292bf 100644 --- a/yearn/v2/vaults.py +++ b/yearn/v2/vaults.py @@ -3,9 +3,9 @@ import re import threading import time -from functools import cached_property from typing import TYPE_CHECKING, Any, Dict, List, Union +from async_property import async_cached_property from brownie import chain from eth_utils import encode_hex, event_abi_to_log_topic from joblib import Parallel, delayed @@ -302,9 +302,8 @@ async def tvl(self, block=None): tvl = total_assets * price / await ERC20(self.vault, asynchronous=True).scale if price else None return Tvl(total_assets, price, tvl) - @cached_property - def _needs_curve_simple(self): - from yearn.prices.curve import curve + @async_cached_property + async def _needs_curve_simple(self): # some curve vaults which should not be calculated with curve logic curve_simple_excludes = { Network.Arbitrum: [ @@ -315,4 +314,4 @@ def _needs_curve_simple(self): if chain.id in curve_simple_excludes: needs_simple = self.vault.address not in curve_simple_excludes[chain.id] - return needs_simple and curve and curve.get_pool(self.token.address) + return needs_simple and magic.curve and await magic.curve.get_pool(self.token.address) From f986d710911fff4b082d2ec6b8eb36cea7b44b5e Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 03/59] chore: remove deprecated code --- yearn/prices/curve.py | 179 +----------------------------------------- yearn/prices/magic.py | 9 +-- 2 files changed, 5 insertions(+), 183 deletions(-) diff --git a/yearn/prices/curve.py b/yearn/prices/curve.py index 37985a084..7484d6f89 100644 --- a/yearn/prices/curve.py +++ b/yearn/prices/curve.py @@ -237,6 +237,7 @@ def get_factory(self, pool: AddressOrContract) -> EthAddress: """ Get metapool factory that has spawned a pool. """ + self.ensure_loaded() try: return next( factory @@ -250,6 +251,7 @@ def get_registry(self, pool: AddressOrContract) -> EthAddress: """ Get registry containing a pool. """ + self.ensure_loaded() try: return next( registry @@ -277,6 +279,7 @@ def get_gauge(self, pool: AddressOrContract, lp_token: AddressOrContract) -> Eth """ Get liquidity gauge address by pool or lp_token. """ + self.ensure_loaded() pool = to_address(pool) lp_token = to_address(lp_token) if chain.id == Network.Mainnet: @@ -298,7 +301,6 @@ def get_gauge(self, pool: AddressOrContract, lp_token: AddressOrContract) -> Eth if gauge != ZERO_ADDRESS: return gauge - @lru_cache(maxsize=None) def get_coins(self, pool: AddressOrContract) -> List[EthAddress]: """ @@ -318,182 +320,7 @@ def get_coins(self, pool: AddressOrContract) -> List[EthAddress]: return [coin for coin in coins if coin not in {None, ZERO_ADDRESS}] - @lru_cache(maxsize=None) - def get_underlying_coins(self, pool: AddressOrContract) -> List[EthAddress]: - pool = to_address(pool) - factory = self.get_factory(pool) - registry = self.get_registry(pool) - - if factory: - factory = contract(factory) - # new factory reverts for non-meta pools - if not hasattr(factory, 'is_meta') or factory.is_meta(pool): - if hasattr(factory, 'get_underlying_coins'): - coins = factory.get_underlying_coins(pool) - elif hasattr(factory, 'get_coins'): - coins = factory.get_coins(pool) - else: - coins = {ZERO_ADDRESS} - else: - coins = factory.get_coins(pool) - elif registry: - registry = contract(registry) - if hasattr(registry, 'get_underlying_coins'): - coins = registry.get_underlying_coins(pool) - elif hasattr(registry, 'get_coins'): - coins = registry.get_coins(pool) - - # pool not in registry, not checking for underlying_coins here - if set(coins) == {ZERO_ADDRESS}: - return self.get_coins(pool) - - return [coin for coin in coins if coin != ZERO_ADDRESS] - - @lru_cache(maxsize=None) - def get_decimals(self, pool: AddressOrContract) -> List[int]: - pool = to_address(pool) - factory = self.get_factory(pool) - registry = self.get_registry(pool) - source = contract(factory or registry) - decimals = source.get_decimals(pool) - - # pool not in registry - if not any(decimals): - coins = self.get_coins(pool) - decimals = fetch_multicall( - *[[contract(token), 'decimals'] for token in coins] - ) - - return [dec for dec in decimals if dec != 0] - - def get_balances(self, pool: AddressOrContract, block: Optional[Block] = None, should_raise_err: bool = True) -> Optional[Dict[EthAddress,float]]: - """ - Get {token: balance} of liquidity in the pool. - """ - pool = to_address(pool) - factory = self.get_factory(pool) - registry = self.get_registry(pool) - coins = self.get_coins(pool) - decimals = self.get_decimals(pool) - - try: - source = contract(factory or registry) - balances = source.get_balances(pool, block_identifier=block) - # fallback for historical queries - except ValueError as e: - if str(e) not in [ - 'execution reverted', - 'No data was returned - the call likely reverted' - ]: raise - - balances = fetch_multicall( - *[[contract(pool), 'balances', i] for i, _ in enumerate(coins)], - block=block - ) - - if not any(balances): - if should_raise_err: - raise ValueError(f'could not fetch balances {pool} at {block}') - return None - - return { - coin: balance / 10 ** dec - for coin, balance, dec in zip(coins, balances, decimals) - } - def get_virtual_price(self, pool: Address, block: Optional[Block] = None) -> Optional[float]: - pool = contract(pool) - try: - return pool.get_virtual_price(block_identifier=block) / 1e18 - except ValueError as e: - if str(e) == "execution reverted": - return None - raise - - def get_tvl(self, pool: AddressOrContract, block: Optional[Block] = None) -> float: - """ - Get total value in Curve pool. - """ - pool = to_address(pool) - balances = self.get_balances(pool, block=block) - - return sum( - amount * magic.get_price(coin, block=block) - for coin, amount in balances.items() - ) - - @ttl_cache(maxsize=None, ttl=600) - def get_price(self, token: AddressOrContract, block: Optional[Block] = None) -> Optional[float]: - token = to_address(token) - pool = self.get_pool(token) - # crypto pools can have different tokens, use slow method - try: - tvl = self.get_tvl(pool, block=block) - except ValueError: - tvl = 0 - supply = contract(token).totalSupply(block_identifier=block) / 1e18 - if supply == 0: - if tvl > 0: - raise ValueError('curve pool has balance but no supply') - return 0 - return tvl / supply - - def get_coin_price(self, token: AddressOrContract, block: Optional[Block] = None) -> Optional[float]: - - # Select the most appropriate pool - pools = self.coin_to_pools[token] - if not pools: - return - elif len(pools) == 1: - pool = pools[0] - else: - # We need to find the pool with the deepest liquidity - balances = [self.get_balances(pool, block, should_raise_err=False) for pool in pools] - deepest_pool, deepest_bal = None, 0 - for pool, pool_bals in zip(pools, balances): - if pool_bals is None: - continue - if isinstance(pool_bals, Exception): - if str(pool_bals).startswith("could not fetch balances"): - continue - raise pool_bals - for _token, bal in pool_bals.items(): - if _token == token and bal > deepest_bal: - deepest_pool = pool - deepest_bal = bal - pool = deepest_pool - - # Get the index for `token` - coins = self.get_coins(pool) - token_in_ix = [i for i, coin in enumerate(coins) if coin == token][0] - amount_in = 10 ** contract(str(token)).decimals() - if len(coins) == 2: - # this works for most typical metapools - token_out_ix = 0 if token_in_ix == 1 else 1 if token_in_ix == 0 else None - elif len(coins) == 3: - # We will just default to using token 0 until we have a reason to make this more flexible - token_out_ix = 0 if token_in_ix in [1, 2] else 1 if token_in_ix == 0 else None - else: - # TODO: handle this sitch if necessary - return None - - # Get the price for `token` using the selected pool. - try: - dy = contract(pool).get_dy(token_in_ix, token_out_ix, amount_in, block_identifier = block) - except: - return None - - if coins[token_out_ix] == EEE_ADDRESS: - token_out = EEE_ADDRESS - amount_out = dy / 10 ** 18 - else: - token_out = contract(coins[token_out_ix]) - amount_out = dy / 10 ** token_out.decimals() - try: - return amount_out * magic.get_price(token_out, block = block) - except PriceError: - return None - async def calculate_boost(self, gauge: Contract, addr: Address, block: Optional[Block] = None) -> Dict[str,float]: results = await fetch_multicall_async( [gauge, "balanceOf", addr], diff --git a/yearn/prices/magic.py b/yearn/prices/magic.py index 9fb0503d2..7d5c898d2 100644 --- a/yearn/prices/magic.py +++ b/yearn/prices/magic.py @@ -9,7 +9,7 @@ from y.networks import Network from yearn.constants import CRV -from yearn.prices import constants, curve +from yearn.prices import constants from yearn.prices.aave import aave from yearn.prices.balancer import balancer as bal from yearn.prices.band import band @@ -123,7 +123,7 @@ def find_price( elif chain.id == Network.Mainnet: # no liquid market for yveCRV-DAO -> return CRV token price if token == Backscratcher().vault.address and block < 11786563: - if curve.curve and CRV: + if CRV: return get_price(CRV, block=block) # no liquidity for curve pool (yvecrv-f) -> return 0 elif token == "0x7E46fd8a30869aa9ed55af031067Df666EfE87da" and block < 14987514: @@ -133,7 +133,6 @@ def find_price( return 0 markets = [ - curve.curve, compound, fixed_forex, generic_amm, @@ -159,10 +158,6 @@ def find_price( price, underlying = price logger.debug("peel %s %s", price, underlying) return price * get_price(underlying, block=block) - - if price is None and token in curve.curve.coin_to_pools: - logger.debug(f'Curve.get_coin_price -> {price}') - price = curve.curve.get_coin_price(token, block = block) if price is None and return_price_during_vault_downtime: for incident in INCIDENTS[token]: From 4ac8a42fb9f949ce5eda3b5e81b76a959ba1a5ec Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 04/59] chore: bump get_logs_asap concurrency --- yearn/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yearn/events.py b/yearn/events.py index d786193d8..13d0967c8 100644 --- a/yearn/events.py +++ b/yearn/events.py @@ -68,7 +68,7 @@ def get_logs_asap( if verbose > 0: logger.info('fetching %d batches', len(ranges)) - batches = Parallel(1, "threading", verbose=verbose)( + batches = Parallel(8, "threading", verbose=verbose)( delayed(web3.eth.get_logs)(_get_logs_params(addresses, topics, start, end)) for start, end in ranges ) From b9e7278bfceabd3fb99d8862dd4f1a578f52532d Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 05/59] chore: add comment --- yearn/prices/curve.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yearn/prices/curve.py b/yearn/prices/curve.py index 7484d6f89..f93649ebb 100644 --- a/yearn/prices/curve.py +++ b/yearn/prices/curve.py @@ -100,6 +100,7 @@ class Ids(IntEnum): Curve_Tricrypto_Factory = 11 class CurveRegistry(metaclass=Singleton): + # NOTE: before deprecating, figure out why this loads more pools than ypm def __init__(self) -> None: if chain.id not in curve_contracts: raise UnsupportedNetwork("curve is not supported on this network") From 2ff5bdd22a398d6b46b8e62f134d4713324f00d2 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 06/59] feat: load strats in subthread to unblock loop --- yearn/v2/vaults.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/yearn/v2/vaults.py b/yearn/v2/vaults.py index b8e1292bf..4242f6c63 100644 --- a/yearn/v2/vaults.py +++ b/yearn/v2/vaults.py @@ -251,6 +251,7 @@ def process_events(self, events): async def _unpack_results(self, results): results, strategy_descs, price = results + strategies = await run_in_thread(getattr, self, 'strategies') return await run_in_subprocess( _unpack_results, self.vault.address, @@ -260,15 +261,15 @@ async def _unpack_results(self, results): self.scale, price, # must be picklable. - [strategy.unique_name for strategy in self.strategies], + [strategy.unique_name for strategy in strategies], strategy_descs, ) async def describe(self, block=None): - await run_in_thread(self.load_strategies) + strategies = await run_in_thread(getattr, self, 'strategies') results = await asyncio.gather( fetch_multicall_async(*[[self.vault, view] for view in self._views], block=block), - asyncio.gather(*[strategy.describe(block=block) for strategy in self.strategies]), + asyncio.gather(*[strategy.describe(block=block) for strategy in strategies]), get_price_return_exceptions(self.token, block=block) ) return await self._unpack_results(results) From db6eb515aea96428a7aee39a86f4504d3401bcc4 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 07/59] feat: drome apy previews --- scripts/drome_apy_previews.py | 242 ++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 scripts/drome_apy_previews.py diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py new file mode 100644 index 000000000..16e38cbea --- /dev/null +++ b/scripts/drome_apy_previews.py @@ -0,0 +1,242 @@ + +""" +This script produces a list of velodrome/aerodrome gauges for which vaults can be created +""" +import asyncio +import dataclasses +import json +import logging +import os +import shutil +import traceback +from datetime import datetime +from pprint import pformat +from time import time +from typing import List, Optional + +import boto3 +import sentry_sdk +from brownie import ZERO_ADDRESS, chain +from multicall.utils import await_awaitable +from tqdm.asyncio import tqdm_asyncio +from y import Contract, Network, magic, ERC20 +from y.exceptions import ContractNotVerified +from y.time import get_block_timestamp_async + +from yearn.apy import Apy, ApyFees, ApyPoints, ApySamples, get_samples +from yearn.apy.common import SECONDS_PER_YEAR +from yearn.apy.curve.simple import Gauge +from yearn.apy.velo import COMPOUNDING +from yearn.debug import Debug +from yearn.exceptions import EmptyS3Export + +logger = logging.getLogger(__name__) +sentry_sdk.set_tag('script','curve_apy_previews') + +sugars = { + Network.Optimism: '', + Network.Base: '0x2073D8035bB2b0F2e85aAF5a8732C6f397F9ff9b', +} + +voters = { + # Velodrome + Network.Optimism: '0x41c914ee0c7e1a5edcd0295623e6dc557b5abf3c', + # Aerodrome + Network.Base: '0x16613524e02ad97eDfeF371bC883F2F5d6C480A5', +} + +def main(): + _upload(await_awaitable(_build_data())) + +async def _build_data(): + start = int(time()) + samples = get_samples() + data = await tqdm_asyncio.gather(*[_build_data_for_lp(lp, samples) for lp in await _get_lps()]) + print(data) + for d in data: + d['updated'] = start + return data + +async def _get_lps() -> List[dict]: + if chain.id not in sugars: + raise ValueError(f"can't get balancer gauges for unsupported network: {chain.id}") + sugar_oracle = await Contract.coroutine(sugars[chain.id]) + return [lp for lp in await sugar_oracle.all.coroutine(999999999999999999999, 0, ZERO_ADDRESS)] + +async def _build_data_for_lp(lp: dict, samples: ApySamples) -> dict: + lp_token = lp[0] + gauge_name = lp[1] + + try: + gauge = await _load_gauge(lp) + except ContractNotVerified as e: + return { + "gauge_name": gauge_name, + "apy": dataclasses.asdict(Apy("error:unverified", 0, 0, ApyFees(0, 0), ApyPoints(0, 0, 0), error_reason=str(e))), + "block": samples.now, + } + + apy_error = Apy("error", 0, 0, ApyFees(0, 0), ApyPoints(0, 0, 0)) + try: + + if gauge.gauge_weight > 0: + apy = await _staking_apy(lp, gauge.gauge, samples) + else: + apy = Apy("zero_weight", 0, 0, ApyFees(0, 0), ApyPoints(0, 0, 0)) + except Exception as error: + apy_error.error_reason = ":".join(str(arg) for arg in error.args) + logger.error(error) + logger.error(gauge) + apy = apy_error + + return { + "gauge_name": gauge_name, + "gauge_address": str(gauge.gauge), + "token0": lp[5], + "token1": lp[8], + "lp_token": lp_token, + "weight": str(gauge.gauge_weight), + "inflation_rate": str(gauge.gauge_inflation_rate), + "working_supply": str(gauge.gauge_working_supply), + "apy": dataclasses.asdict(apy), + "block": samples.now, + } + +async def _load_gauge(lp: dict) -> Gauge: + lp_address = lp[0] + gauge_address = lp[11] + if gauge_address == ZERO_ADDRESS: + logger.warning("lp %s has no gauge", lp_address) + raise ContractNotVerified(ZERO_ADDRESS) + voter = await Contract.coroutine(voters[chain.id]) + pool, gauge, weight = await asyncio.gather( + Contract.coroutine(lp_address), + Contract.coroutine(gauge_address), + voter.weights.coroutine(lp_address), + ) + inflation_rate, working_supply = await asyncio.gather(gauge.rewardRate.coroutine(), gauge.totalSupply.coroutine()) + return Gauge(lp_address, pool, gauge, weight, inflation_rate, working_supply) + +async def _staking_apy(lp: dict, staking_rewards: Contract, samples: ApySamples, block: Optional[int]=None) -> float: + + current_time = time() if block is None else await get_block_timestamp_async(block) + + reward_token, rate, total_supply, end = await asyncio.gather( + staking_rewards.rewardToken.coroutine(block_identifier=block), + staking_rewards.rewardRate.coroutine(block_identifier=block), + staking_rewards.totalSupply.coroutine(block_identifier=block), + staking_rewards.periodFinish.coroutine(block_identifier=block), + ) + + # NOTE: should perf be 0? + #performance = await vault.vault.performanceFee.coroutine(block_identifier=block) / 1e4 if hasattr(vault.vault, "performanceFee") else 0 + performance = 0 + # NOTE: should mgmt be 0? + #management = await vault.vault.managementFee.coroutine(block_identifier=block) / 1e4 if hasattr(vault.vault, "managementFee") else 0 + management = 0 + # since its a fork we still call it keepVELO + #keep = await vault.strategies[0].strategy.localKeepVELO.coroutine(block_identifier=block) / 1e4 if hasattr(vault.strategies[0].strategy, "localKeepVELO") else 0 + # NOTE: should keep be 0? + keep = 0 + rate = rate * (1 - keep) + fees = ApyFees(performance=performance, management=management, keep_velo=keep) + + if end < current_time or total_supply == 0 or rate == 0: + return Apy("v2:aero_unpopular", gross_apr=0, net_apy=0, fees=fees) + + pool_price, token_price = await asyncio.gather( + magic.get_price(lp[0], block=block, sync=False), + magic.get_price(reward_token, block=block, sync=False), + ) + + gross_apr = (SECONDS_PER_YEAR * (rate / 1e18) * token_price) / (pool_price * (total_supply / 1e18)) + + net_apr = gross_apr * (1 - performance) - management + net_apy = (1 + (net_apr / COMPOUNDING)) ** COMPOUNDING - 1 + # NOTE: do we need this? + #staking_rewards_apr = await _get_staking_rewards_apr(reward_token, pool_price, token_price, rate, total_supply, samples) + if os.getenv("DEBUG", None): + logger.info(pformat(Debug().collect_variables(locals()))) + return Apy("v2:aero", gross_apr=gross_apr, net_apy=net_apy, fees=fees) #, staking_rewards_apr=staking_rewards_apr) + +async def _get_staking_rewards_apr(reward_token: str, lp_price: float, reward_price: float, reward_rate: int, total_supply_staked: int, samples: ApySamples): + vault_scale = 10 ** 18 + reward_token_scale = await ERC20(reward_token, asynchronous=True).scale + per_staking_token_rate = (reward_rate / reward_token_scale) / (total_supply_staked / vault_scale) + rewards_vault_apy = (await rewards_vault.apy(samples)).net_apy + emissions_apr = SECONDS_PER_YEAR * per_staking_token_rate * reward_price / lp_price + return emissions_apr * (1 + rewards_vault_apy) + +def _upload(data): + print(json.dumps(data, sort_keys=True, indent=4)) + + file_name, s3_path = _get_export_paths("curve-factory") + with open(file_name, "w+") as f: + json.dump(data, f) + + if os.getenv("DEBUG", None): + return + + aws_bucket = os.environ.get("AWS_BUCKET") + + s3 = _get_s3() + s3.upload_file( + file_name, + aws_bucket, + s3_path, + ExtraArgs={'ContentType': "application/json", 'CacheControl': "max-age=1800"}, + ) + + +def _get_s3(): + aws_key = os.environ.get("AWS_ACCESS_KEY") + aws_secret = os.environ.get("AWS_ACCESS_SECRET") + + kwargs = {} + if aws_key is not None: + kwargs["aws_access_key_id"] = aws_key + if aws_secret is not None: + kwargs["aws_secret_access_key"] = aws_secret + + return boto3.client("s3", **kwargs) + + +def _get_export_paths(suffix): + out = "generated" + if os.path.isdir(out): + shutil.rmtree(out) + os.makedirs(out, exist_ok=True) + + api_path = os.path.join("v1", "chains", f"{chain.id}", "apy-previews") + + file_base_path = os.path.join(out, api_path) + os.makedirs(file_base_path, exist_ok=True) + + file_name = os.path.join(file_base_path, suffix) + s3_path = os.path.join(api_path, suffix) + return file_name, s3_path + +def with_monitoring(): + if os.getenv("DEBUG", None): + main() + return + from telegram.ext import Updater + + private_group = os.environ.get('TG_YFIREBOT_GROUP_INTERNAL') + public_group = os.environ.get('TG_YFIREBOT_GROUP_EXTERNAL') + updater = Updater(os.environ.get('TG_YFIREBOT')) + now = datetime.now() + message = f"`[{now}]`\nāš™ļø Curve Previews API for {Network.name()} is updating..." + ping = updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown") + ping = ping.message_id + try: + main() + except Exception as error: + tb = traceback.format_exc() + now = datetime.now() + message = f"`[{now}]`\nšŸ”„ Curve Previews API update for {Network.name()} failed!\n```\n{tb}\n```"[:4000] + updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown", reply_to_message_id=ping) + updater.bot.send_message(chat_id=public_group, text=message, parse_mode="Markdown") + raise error + message = f"āœ… Curve Previews API update for {Network.name()} successful!" + updater.bot.send_message(chat_id=private_group, text=message, reply_to_message_id=ping) \ No newline at end of file From fd08797f930b1ac3a2c33bfba642ea4d6e5e28d6 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 08/59] feat: cleanup for prod --- scripts/drome_apy_previews.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py index 16e38cbea..b262cb4d8 100644 --- a/scripts/drome_apy_previews.py +++ b/scripts/drome_apy_previews.py @@ -2,6 +2,7 @@ """ This script produces a list of velodrome/aerodrome gauges for which vaults can be created """ + import asyncio import dataclasses import json @@ -28,13 +29,12 @@ from yearn.apy.curve.simple import Gauge from yearn.apy.velo import COMPOUNDING from yearn.debug import Debug -from yearn.exceptions import EmptyS3Export logger = logging.getLogger(__name__) sentry_sdk.set_tag('script','curve_apy_previews') sugars = { - Network.Optimism: '', + Network.Optimism: '0x4D996E294B00cE8287C16A2b9A4e637ecA5c939f', Network.Base: '0x2073D8035bB2b0F2e85aAF5a8732C6f397F9ff9b', } @@ -51,10 +51,10 @@ def main(): async def _build_data(): start = int(time()) samples = get_samples() - data = await tqdm_asyncio.gather(*[_build_data_for_lp(lp, samples) for lp in await _get_lps()]) - print(data) + data = [d for d in await tqdm_asyncio.gather(*[_build_data_for_lp(lp, samples) for lp in await _get_lps()]) if d] for d in data: d['updated'] = start + print(data) return data async def _get_lps() -> List[dict]: @@ -63,12 +63,17 @@ async def _get_lps() -> List[dict]: sugar_oracle = await Contract.coroutine(sugars[chain.id]) return [lp for lp in await sugar_oracle.all.coroutine(999999999999999999999, 0, ZERO_ADDRESS)] -async def _build_data_for_lp(lp: dict, samples: ApySamples) -> dict: +class NoGaugeFound(Exception): + pass + +async def _build_data_for_lp(lp: dict, samples: ApySamples) -> Optional[dict]: lp_token = lp[0] gauge_name = lp[1] try: gauge = await _load_gauge(lp) + except NoGaugeFound: + return None except ContractNotVerified as e: return { "gauge_name": gauge_name, @@ -106,8 +111,7 @@ async def _load_gauge(lp: dict) -> Gauge: lp_address = lp[0] gauge_address = lp[11] if gauge_address == ZERO_ADDRESS: - logger.warning("lp %s has no gauge", lp_address) - raise ContractNotVerified(ZERO_ADDRESS) + raise NoGaugeFound(f"lp {lp_address} has no gauge") voter = await Contract.coroutine(voters[chain.id]) pool, gauge, weight = await asyncio.gather( Contract.coroutine(lp_address), From ebf8149abe39ab1b447c124c2291039788b52929 Mon Sep 17 00:00:00 2001 From: 0xBasically <0xBasic@yearn.finance> Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 09/59] update make --- Makefile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Makefile b/Makefile index 86301a59a..bc381ea21 100644 --- a/Makefile +++ b/Makefile @@ -275,6 +275,18 @@ apy-yeth-monitoring: apy-yeth: make up commands="yeth" network=eth filter=yeth + +aerodrome-apy-previews: + make up commands="drome_apy_previews" network=base + +aerodrome-apy-previews-monitoring: + make up commands="drome_apy_previews with_monitoring" network=base + +velodrome-apy-previews: + make up commands="drome_apy_previews" network=optimism + +velodrome-apy-previews-monitoring: + make up commands="drome_apy_previews with_monitoring" network=optimism # revenue scripts revenues: From 0292d91f28aea324226539df699873980c9828d7 Mon Sep 17 00:00:00 2001 From: 0xBasically <0xBasic@yearn.finance> Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 10/59] update telegram msg --- scripts/drome_apy_previews.py | 46 ++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py index b262cb4d8..d679ca8d8 100644 --- a/scripts/drome_apy_previews.py +++ b/scripts/drome_apy_previews.py @@ -230,17 +230,35 @@ def with_monitoring(): public_group = os.environ.get('TG_YFIREBOT_GROUP_EXTERNAL') updater = Updater(os.environ.get('TG_YFIREBOT')) now = datetime.now() - message = f"`[{now}]`\nāš™ļø Curve Previews API for {Network.name()} is updating..." - ping = updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown") - ping = ping.message_id - try: - main() - except Exception as error: - tb = traceback.format_exc() - now = datetime.now() - message = f"`[{now}]`\nšŸ”„ Curve Previews API update for {Network.name()} failed!\n```\n{tb}\n```"[:4000] - updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown", reply_to_message_id=ping) - updater.bot.send_message(chat_id=public_group, text=message, parse_mode="Markdown") - raise error - message = f"āœ… Curve Previews API update for {Network.name()} successful!" - updater.bot.send_message(chat_id=private_group, text=message, reply_to_message_id=ping) \ No newline at end of file + if Network.name() == "Optimism": + message = f"`[{now}]`\nāš™ļø Velodrome Previews API for {Network.name()} is updating..." + ping = updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown") + ping = ping.message_id + try: + main() + except Exception as error: + tb = traceback.format_exc() + now = datetime.now() + message = f"`[{now}]`\nšŸ”„ Velodrome Previews API update for {Network.name()} failed!\n```\n{tb}\n```"[:4000] + updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown", reply_to_message_id=ping) + updater.bot.send_message(chat_id=public_group, text=message, parse_mode="Markdown") + raise error + message = f"āœ… Velodrome Previews API update for {Network.name()} successful!" + updater.bot.send_message(chat_id=private_group, text=message, reply_to_message_id=ping) + elif Network.name() == "Base": + message = f"`[{now}]`\nāš™ļø Aerodrome Previews API for {Network.name()} is updating..." + ping = updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown") + ping = ping.message_id + try: + main() + except Exception as error: + tb = traceback.format_exc() + now = datetime.now() + message = f"`[{now}]`\nšŸ”„ Aerodrome Previews API update for {Network.name()} failed!\n```\n{tb}\n```"[:4000] + updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown", reply_to_message_id=ping) + updater.bot.send_message(chat_id=public_group, text=message, parse_mode="Markdown") + raise error + message = f"āœ… Aerodrome Previews API update for {Network.name()} successful!" + updater.bot.send_message(chat_id=private_group, text=message, reply_to_message_id=ping) + else: + message = f"{Network.name()} network not a valid network for previews script." \ No newline at end of file From 5238e858aac158b41eb385f73c7a3723b0628d5e Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 11/59] fix: labels --- scripts/drome_apy_previews.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py index d679ca8d8..12b227721 100644 --- a/scripts/drome_apy_previews.py +++ b/scripts/drome_apy_previews.py @@ -45,6 +45,11 @@ Network.Base: '0x16613524e02ad97eDfeF371bC883F2F5d6C480A5', } +labels = { + Network.Optimism: 'velo', + Network.Base: 'aero', +} + def main(): _upload(await_awaitable(_build_data())) @@ -146,7 +151,7 @@ async def _staking_apy(lp: dict, staking_rewards: Contract, samples: ApySamples, fees = ApyFees(performance=performance, management=management, keep_velo=keep) if end < current_time or total_supply == 0 or rate == 0: - return Apy("v2:aero_unpopular", gross_apr=0, net_apy=0, fees=fees) + return Apy(f"v2:{labels[chain.id]}_unpopular", gross_apr=0, net_apy=0, fees=fees) pool_price, token_price = await asyncio.gather( magic.get_price(lp[0], block=block, sync=False), @@ -161,8 +166,10 @@ async def _staking_apy(lp: dict, staking_rewards: Contract, samples: ApySamples, #staking_rewards_apr = await _get_staking_rewards_apr(reward_token, pool_price, token_price, rate, total_supply, samples) if os.getenv("DEBUG", None): logger.info(pformat(Debug().collect_variables(locals()))) - return Apy("v2:aero", gross_apr=gross_apr, net_apy=net_apy, fees=fees) #, staking_rewards_apr=staking_rewards_apr) + return Apy(f"v2:{labels[chain.id]}", gross_apr=gross_apr, net_apy=net_apy, fees=fees) #, staking_rewards_apr=staking_rewards_apr) +# NOTE: do we need this? +""" async def _get_staking_rewards_apr(reward_token: str, lp_price: float, reward_price: float, reward_rate: int, total_supply_staked: int, samples: ApySamples): vault_scale = 10 ** 18 reward_token_scale = await ERC20(reward_token, asynchronous=True).scale @@ -170,6 +177,7 @@ async def _get_staking_rewards_apr(reward_token: str, lp_price: float, reward_pr rewards_vault_apy = (await rewards_vault.apy(samples)).net_apy emissions_apr = SECONDS_PER_YEAR * per_staking_token_rate * reward_price / lp_price return emissions_apr * (1 + rewards_vault_apy) +""" def _upload(data): print(json.dumps(data, sort_keys=True, indent=4)) From af440eedb6f7fd45df5201bbe4c3279995021856 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 12/59] chore: refactor --- scripts/drome_apy_previews.py | 55 ++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py index 12b227721..66cd69376 100644 --- a/scripts/drome_apy_previews.py +++ b/scripts/drome_apy_previews.py @@ -18,9 +18,10 @@ import boto3 import sentry_sdk from brownie import ZERO_ADDRESS, chain +from msgspec import Struct from multicall.utils import await_awaitable from tqdm.asyncio import tqdm_asyncio -from y import Contract, Network, magic, ERC20 +from y import ERC20, Contract, Network, magic from y.exceptions import ContractNotVerified from y.time import get_block_timestamp_async @@ -33,22 +34,33 @@ logger = logging.getLogger(__name__) sentry_sdk.set_tag('script','curve_apy_previews') -sugars = { - Network.Optimism: '0x4D996E294B00cE8287C16A2b9A4e637ecA5c939f', - Network.Base: '0x2073D8035bB2b0F2e85aAF5a8732C6f397F9ff9b', -} +class Drome(Struct): + label: str + sugar: str + voter: str + # A random vault to check fees + fee_checker: str -voters = { - # Velodrome - Network.Optimism: '0x41c914ee0c7e1a5edcd0295623e6dc557b5abf3c', - # Aerodrome - Network.Base: '0x16613524e02ad97eDfeF371bC883F2F5d6C480A5', -} +class NoGaugeFound(Exception): + pass -labels = { - Network.Optimism: 'velo', - Network.Base: 'aero', -} +try: + drome = { + Network.Optimism: Drome( + label='velo', + sugar='0x4D996E294B00cE8287C16A2b9A4e637ecA5c939f', + voter='0x41c914ee0c7e1a5edcd0295623e6dc557b5abf3c', + fee_checker='0xbC61B71562b01a3a4808D3B9291A3Bf743AB3361', + ), + Network.Base: Drome( + label='aero', + sugar='0x2073D8035bB2b0F2e85aAF5a8732C6f397F9ff9b', + voter='0x16613524e02ad97eDfeF371bC883F2F5d6C480A5', + fee_checker='0xEcFc1e5BDa4d4191c9Cab053ec704347Db87Be5d', + ), + }[chain.id] +except KeyError: + raise ValueError(f"there is no drome on unsupported network: {chain.id}") def main(): _upload(await_awaitable(_build_data())) @@ -63,14 +75,9 @@ async def _build_data(): return data async def _get_lps() -> List[dict]: - if chain.id not in sugars: - raise ValueError(f"can't get balancer gauges for unsupported network: {chain.id}") - sugar_oracle = await Contract.coroutine(sugars[chain.id]) + sugar_oracle = await Contract.coroutine(drome.sugar) return [lp for lp in await sugar_oracle.all.coroutine(999999999999999999999, 0, ZERO_ADDRESS)] -class NoGaugeFound(Exception): - pass - async def _build_data_for_lp(lp: dict, samples: ApySamples) -> Optional[dict]: lp_token = lp[0] gauge_name = lp[1] @@ -117,7 +124,7 @@ async def _load_gauge(lp: dict) -> Gauge: gauge_address = lp[11] if gauge_address == ZERO_ADDRESS: raise NoGaugeFound(f"lp {lp_address} has no gauge") - voter = await Contract.coroutine(voters[chain.id]) + voter = await Contract.coroutine(drome.voter) pool, gauge, weight = await asyncio.gather( Contract.coroutine(lp_address), Contract.coroutine(gauge_address), @@ -151,7 +158,7 @@ async def _staking_apy(lp: dict, staking_rewards: Contract, samples: ApySamples, fees = ApyFees(performance=performance, management=management, keep_velo=keep) if end < current_time or total_supply == 0 or rate == 0: - return Apy(f"v2:{labels[chain.id]}_unpopular", gross_apr=0, net_apy=0, fees=fees) + return Apy(f"v2:{drome.label}_unpopular", gross_apr=0, net_apy=0, fees=fees) pool_price, token_price = await asyncio.gather( magic.get_price(lp[0], block=block, sync=False), @@ -166,7 +173,7 @@ async def _staking_apy(lp: dict, staking_rewards: Contract, samples: ApySamples, #staking_rewards_apr = await _get_staking_rewards_apr(reward_token, pool_price, token_price, rate, total_supply, samples) if os.getenv("DEBUG", None): logger.info(pformat(Debug().collect_variables(locals()))) - return Apy(f"v2:{labels[chain.id]}", gross_apr=gross_apr, net_apy=net_apy, fees=fees) #, staking_rewards_apr=staking_rewards_apr) + return Apy(f"v2:{drome.label}", gross_apr=gross_apr, net_apy=net_apy, fees=fees) #, staking_rewards_apr=staking_rewards_apr) # NOTE: do we need this? """ From c73a26b6a3276fa12c7c2643de7c29d42c3dcd81 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 13/59] feat: factor in fees --- scripts/drome_apy_previews.py | 61 ++++++++++++++++------------------- 1 file changed, 27 insertions(+), 34 deletions(-) diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py index 66cd69376..a43cf7431 100644 --- a/scripts/drome_apy_previews.py +++ b/scripts/drome_apy_previews.py @@ -25,7 +25,7 @@ from y.exceptions import ContractNotVerified from y.time import get_block_timestamp_async -from yearn.apy import Apy, ApyFees, ApyPoints, ApySamples, get_samples +from yearn.apy import Apy, ApyFees, ApyPoints, get_samples from yearn.apy.common import SECONDS_PER_YEAR from yearn.apy.curve.simple import Gauge from yearn.apy.velo import COMPOUNDING @@ -35,6 +35,7 @@ sentry_sdk.set_tag('script','curve_apy_previews') class Drome(Struct): + """Holds various params for a drome deployment""" label: str sugar: str voter: str @@ -62,44 +63,48 @@ class NoGaugeFound(Exception): except KeyError: raise ValueError(f"there is no drome on unsupported network: {chain.id}") +fee_checker = Contract(drome.fee_checker) +performance_fee = fee_checker.performanceFee() / 1e4 +management_fee = fee_checker.managementFee() / 1e4 +fee_checker_strat = Contract(fee_checker.withdrawalQueue(0)) + +keep = fee_checker_strat.localKeepVELO() / 1e4 +unkeep = 1 - keep + def main(): _upload(await_awaitable(_build_data())) async def _build_data(): start = int(time()) - samples = get_samples() - data = [d for d in await tqdm_asyncio.gather(*[_build_data_for_lp(lp, samples) for lp in await _get_lps()]) if d] + block = get_samples().now + data = [d for d in await tqdm_asyncio.gather(*[_build_data_for_lp(lp, block) for lp in await _get_lps(block)]) if d] for d in data: d['updated'] = start print(data) return data -async def _get_lps() -> List[dict]: +async def _get_lps(block: Optional[int] = None) -> List[dict]: sugar_oracle = await Contract.coroutine(drome.sugar) - return [lp for lp in await sugar_oracle.all.coroutine(999999999999999999999, 0, ZERO_ADDRESS)] + return [lp for lp in await sugar_oracle.all.coroutine(999999999999999999999, 0, ZERO_ADDRESS, block_identifier=block)] -async def _build_data_for_lp(lp: dict, samples: ApySamples) -> Optional[dict]: +async def _build_data_for_lp(lp: dict, block: Optional[int] = None) -> Optional[dict]: lp_token = lp[0] gauge_name = lp[1] try: - gauge = await _load_gauge(lp) + gauge = await _load_gauge(lp, block=block) except NoGaugeFound: return None except ContractNotVerified as e: return { "gauge_name": gauge_name, "apy": dataclasses.asdict(Apy("error:unverified", 0, 0, ApyFees(0, 0), ApyPoints(0, 0, 0), error_reason=str(e))), - "block": samples.now, + "block": block, } apy_error = Apy("error", 0, 0, ApyFees(0, 0), ApyPoints(0, 0, 0)) try: - - if gauge.gauge_weight > 0: - apy = await _staking_apy(lp, gauge.gauge, samples) - else: - apy = Apy("zero_weight", 0, 0, ApyFees(0, 0), ApyPoints(0, 0, 0)) + apy = await _staking_apy(lp, gauge.gauge, block=block) if gauge.gauge_weight > 0 else Apy("zero_weight", 0, 0, ApyFees(0, 0), ApyPoints(0, 0, 0)) except Exception as error: apy_error.error_reason = ":".join(str(arg) for arg in error.args) logger.error(error) @@ -116,10 +121,10 @@ async def _build_data_for_lp(lp: dict, samples: ApySamples) -> Optional[dict]: "inflation_rate": str(gauge.gauge_inflation_rate), "working_supply": str(gauge.gauge_working_supply), "apy": dataclasses.asdict(apy), - "block": samples.now, + "block": block, } -async def _load_gauge(lp: dict) -> Gauge: +async def _load_gauge(lp: dict, block: Optional[int] = None) -> Gauge: lp_address = lp[0] gauge_address = lp[11] if gauge_address == ZERO_ADDRESS: @@ -128,12 +133,12 @@ async def _load_gauge(lp: dict) -> Gauge: pool, gauge, weight = await asyncio.gather( Contract.coroutine(lp_address), Contract.coroutine(gauge_address), - voter.weights.coroutine(lp_address), + voter.weights.coroutine(lp_address, block_identifier=block), ) - inflation_rate, working_supply = await asyncio.gather(gauge.rewardRate.coroutine(), gauge.totalSupply.coroutine()) + inflation_rate, working_supply = await asyncio.gather(gauge.rewardRate.coroutine(block_identifier=block), gauge.totalSupply.coroutine(block_identifier=block)) return Gauge(lp_address, pool, gauge, weight, inflation_rate, working_supply) -async def _staking_apy(lp: dict, staking_rewards: Contract, samples: ApySamples, block: Optional[int]=None) -> float: +async def _staking_apy(lp: dict, staking_rewards: Contract, block: Optional[int]=None) -> float: current_time = time() if block is None else await get_block_timestamp_async(block) @@ -144,18 +149,8 @@ async def _staking_apy(lp: dict, staking_rewards: Contract, samples: ApySamples, staking_rewards.periodFinish.coroutine(block_identifier=block), ) - # NOTE: should perf be 0? - #performance = await vault.vault.performanceFee.coroutine(block_identifier=block) / 1e4 if hasattr(vault.vault, "performanceFee") else 0 - performance = 0 - # NOTE: should mgmt be 0? - #management = await vault.vault.managementFee.coroutine(block_identifier=block) / 1e4 if hasattr(vault.vault, "managementFee") else 0 - management = 0 - # since its a fork we still call it keepVELO - #keep = await vault.strategies[0].strategy.localKeepVELO.coroutine(block_identifier=block) / 1e4 if hasattr(vault.strategies[0].strategy, "localKeepVELO") else 0 - # NOTE: should keep be 0? - keep = 0 - rate = rate * (1 - keep) - fees = ApyFees(performance=performance, management=management, keep_velo=keep) + rate *= unkeep + fees = ApyFees(performance=performance_fee, management=management_fee, keep_velo=keep) if end < current_time or total_supply == 0 or rate == 0: return Apy(f"v2:{drome.label}_unpopular", gross_apr=0, net_apy=0, fees=fees) @@ -167,13 +162,11 @@ async def _staking_apy(lp: dict, staking_rewards: Contract, samples: ApySamples, gross_apr = (SECONDS_PER_YEAR * (rate / 1e18) * token_price) / (pool_price * (total_supply / 1e18)) - net_apr = gross_apr * (1 - performance) - management + net_apr = gross_apr * (1 - performance_fee) - management_fee net_apy = (1 + (net_apr / COMPOUNDING)) ** COMPOUNDING - 1 - # NOTE: do we need this? - #staking_rewards_apr = await _get_staking_rewards_apr(reward_token, pool_price, token_price, rate, total_supply, samples) if os.getenv("DEBUG", None): logger.info(pformat(Debug().collect_variables(locals()))) - return Apy(f"v2:{drome.label}", gross_apr=gross_apr, net_apy=net_apy, fees=fees) #, staking_rewards_apr=staking_rewards_apr) + return Apy(f"v2:{drome.label}", gross_apr=gross_apr, net_apy=net_apy, fees=fees) # NOTE: do we need this? """ From c6cf9adb0e0205ba5a5c6707cf362f02fdf9b8ef Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 14/59] feat: filter out lps with vaults --- scripts/drome_apy_previews.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py index a43cf7431..2eeadad6b 100644 --- a/scripts/drome_apy_previews.py +++ b/scripts/drome_apy_previews.py @@ -29,6 +29,7 @@ from yearn.apy.common import SECONDS_PER_YEAR from yearn.apy.curve.simple import Gauge from yearn.apy.velo import COMPOUNDING +from yearn.v2.registry import Registry from yearn.debug import Debug logger = logging.getLogger(__name__) @@ -42,9 +43,6 @@ class Drome(Struct): # A random vault to check fees fee_checker: str -class NoGaugeFound(Exception): - pass - try: drome = { Network.Optimism: Drome( @@ -77,15 +75,17 @@ def main(): async def _build_data(): start = int(time()) block = get_samples().now - data = [d for d in await tqdm_asyncio.gather(*[_build_data_for_lp(lp, block) for lp in await _get_lps(block)]) if d] + data = [d for d in await tqdm_asyncio.gather(*[_build_data_for_lp(lp, block) for lp in await _get_lps_with_vault_potential()]) if d] for d in data: d['updated'] = start print(data) return data -async def _get_lps(block: Optional[int] = None) -> List[dict]: +async def _get_lps_with_vault_potential() -> List[dict]: sugar_oracle = await Contract.coroutine(drome.sugar) - return [lp for lp in await sugar_oracle.all.coroutine(999999999999999999999, 0, ZERO_ADDRESS, block_identifier=block)] + current_vaults = Registry(watch_events_forever=False, include_experimental=False).vaults + current_underlyings = [str(vault.token) for vault in current_vaults] + return [lp for lp in await sugar_oracle.all.coroutine(999999999999999999999, 0, ZERO_ADDRESS) if lp[0] not in current_underlyings and lp[11] != ZERO_ADDRESS] async def _build_data_for_lp(lp: dict, block: Optional[int] = None) -> Optional[dict]: lp_token = lp[0] @@ -93,8 +93,6 @@ async def _build_data_for_lp(lp: dict, block: Optional[int] = None) -> Optional[ try: gauge = await _load_gauge(lp, block=block) - except NoGaugeFound: - return None except ContractNotVerified as e: return { "gauge_name": gauge_name, @@ -127,15 +125,16 @@ async def _build_data_for_lp(lp: dict, block: Optional[int] = None) -> Optional[ async def _load_gauge(lp: dict, block: Optional[int] = None) -> Gauge: lp_address = lp[0] gauge_address = lp[11] - if gauge_address == ZERO_ADDRESS: - raise NoGaugeFound(f"lp {lp_address} has no gauge") voter = await Contract.coroutine(drome.voter) pool, gauge, weight = await asyncio.gather( Contract.coroutine(lp_address), Contract.coroutine(gauge_address), voter.weights.coroutine(lp_address, block_identifier=block), ) - inflation_rate, working_supply = await asyncio.gather(gauge.rewardRate.coroutine(block_identifier=block), gauge.totalSupply.coroutine(block_identifier=block)) + inflation_rate, working_supply = await asyncio.gather( + gauge.rewardRate.coroutine(block_identifier=block), + gauge.totalSupply.coroutine(block_identifier=block), + ) return Gauge(lp_address, pool, gauge, weight, inflation_rate, working_supply) async def _staking_apy(lp: dict, staking_rewards: Contract, block: Optional[int]=None) -> float: From f275b1e12a55cdb176aa46cbb6fe698aa5d9b272 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 15/59] chore: cleanup --- scripts/drome_apy_previews.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py index 2eeadad6b..9068c7018 100644 --- a/scripts/drome_apy_previews.py +++ b/scripts/drome_apy_previews.py @@ -167,17 +167,6 @@ async def _staking_apy(lp: dict, staking_rewards: Contract, block: Optional[int] logger.info(pformat(Debug().collect_variables(locals()))) return Apy(f"v2:{drome.label}", gross_apr=gross_apr, net_apy=net_apy, fees=fees) -# NOTE: do we need this? -""" -async def _get_staking_rewards_apr(reward_token: str, lp_price: float, reward_price: float, reward_rate: int, total_supply_staked: int, samples: ApySamples): - vault_scale = 10 ** 18 - reward_token_scale = await ERC20(reward_token, asynchronous=True).scale - per_staking_token_rate = (reward_rate / reward_token_scale) / (total_supply_staked / vault_scale) - rewards_vault_apy = (await rewards_vault.apy(samples)).net_apy - emissions_apr = SECONDS_PER_YEAR * per_staking_token_rate * reward_price / lp_price - return emissions_apr * (1 + rewards_vault_apy) -""" - def _upload(data): print(json.dumps(data, sort_keys=True, indent=4)) From dd775d05976f44f32274a66b644a22c138446eb0 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 16/59] fix: ApyFees --- scripts/drome_apy_previews.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py index 9068c7018..cde3ca1b1 100644 --- a/scripts/drome_apy_previews.py +++ b/scripts/drome_apy_previews.py @@ -69,6 +69,8 @@ class Drome(Struct): keep = fee_checker_strat.localKeepVELO() / 1e4 unkeep = 1 - keep +fees = ApyFees(performance=performance_fee, management=management_fee, keep_velo=keep) + def main(): _upload(await_awaitable(_build_data())) @@ -96,13 +98,13 @@ async def _build_data_for_lp(lp: dict, block: Optional[int] = None) -> Optional[ except ContractNotVerified as e: return { "gauge_name": gauge_name, - "apy": dataclasses.asdict(Apy("error:unverified", 0, 0, ApyFees(0, 0), ApyPoints(0, 0, 0), error_reason=str(e))), + "apy": dataclasses.asdict(Apy("error:unverified", 0, 0, fees, ApyPoints(0, 0, 0), error_reason=str(e))), "block": block, } - apy_error = Apy("error", 0, 0, ApyFees(0, 0), ApyPoints(0, 0, 0)) + apy_error = Apy("error", 0, 0, fees, ApyPoints(0, 0, 0)) try: - apy = await _staking_apy(lp, gauge.gauge, block=block) if gauge.gauge_weight > 0 else Apy("zero_weight", 0, 0, ApyFees(0, 0), ApyPoints(0, 0, 0)) + apy = await _staking_apy(lp, gauge.gauge, block=block) if gauge.gauge_weight > 0 else Apy("zero_weight", 0, 0, fees, ApyPoints(0, 0, 0)) except Exception as error: apy_error.error_reason = ":".join(str(arg) for arg in error.args) logger.error(error) @@ -149,7 +151,6 @@ async def _staking_apy(lp: dict, staking_rewards: Contract, block: Optional[int] ) rate *= unkeep - fees = ApyFees(performance=performance_fee, management=management_fee, keep_velo=keep) if end < current_time or total_supply == 0 or rate == 0: return Apy(f"v2:{drome.label}_unpopular", gross_apr=0, net_apy=0, fees=fees) From d0d02f7ef0d1c1e934515ff102cc84c70fcfff2f Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 17/59] chore: refactor out points --- scripts/drome_apy_previews.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py index cde3ca1b1..d36aae65c 100644 --- a/scripts/drome_apy_previews.py +++ b/scripts/drome_apy_previews.py @@ -25,7 +25,7 @@ from y.exceptions import ContractNotVerified from y.time import get_block_timestamp_async -from yearn.apy import Apy, ApyFees, ApyPoints, get_samples +from yearn.apy import Apy, ApyFees, get_samples from yearn.apy.common import SECONDS_PER_YEAR from yearn.apy.curve.simple import Gauge from yearn.apy.velo import COMPOUNDING @@ -98,18 +98,16 @@ async def _build_data_for_lp(lp: dict, block: Optional[int] = None) -> Optional[ except ContractNotVerified as e: return { "gauge_name": gauge_name, - "apy": dataclasses.asdict(Apy("error:unverified", 0, 0, fees, ApyPoints(0, 0, 0), error_reason=str(e))), + "apy": dataclasses.asdict(Apy("error:unverified", 0, 0, fees, error_reason=str(e))), "block": block, } - apy_error = Apy("error", 0, 0, fees, ApyPoints(0, 0, 0)) try: - apy = await _staking_apy(lp, gauge.gauge, block=block) if gauge.gauge_weight > 0 else Apy("zero_weight", 0, 0, fees, ApyPoints(0, 0, 0)) + apy = await _staking_apy(lp, gauge.gauge, block=block) if gauge.gauge_weight > 0 else Apy("zero_weight", 0, 0, fees) except Exception as error: - apy_error.error_reason = ":".join(str(arg) for arg in error.args) logger.error(error) logger.error(gauge) - apy = apy_error + apy = Apy("error", 0, 0, fees, error_reason=":".join(str(arg) for arg in error.args)) return { "gauge_name": gauge_name, From 16d2ffbd8b2fdc995c1b8d25674fa031841966d4 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 18/59] feat: refactor to prep for bal apy previews --- scripts/curve_apy_previews.py | 109 ++------------------------------- scripts/drome_apy_previews.py | 112 ++++------------------------------ yearn/helpers/s3.py | 68 +++++++++++++++++++++ yearn/helpers/telegram.py | 39 ++++++++++++ 4 files changed, 124 insertions(+), 204 deletions(-) create mode 100644 yearn/helpers/s3.py create mode 100644 yearn/helpers/telegram.py diff --git a/scripts/curve_apy_previews.py b/scripts/curve_apy_previews.py index fc9142ca7..edc815619 100644 --- a/scripts/curve_apy_previews.py +++ b/scripts/curve_apy_previews.py @@ -1,25 +1,19 @@ import dataclasses -import json import logging -import os import re -import shutil from time import sleep, time -from datetime import datetime -import traceback -import boto3 import requests import sentry_sdk from brownie import ZERO_ADDRESS, chain from brownie.exceptions import ContractNotFound from multicall.utils import await_awaitable -from y import Contract, Network, PriceError +from y import Contract, Network from y.exceptions import ContractNotVerified -from yearn.apy import Apy, ApyFees, ApyPoints, ApySamples, get_samples +from yearn.apy import Apy, ApyFees, ApyPoints, get_samples from yearn.apy.curve.simple import Gauge, calculate_simple -from yearn.exceptions import EmptyS3Export +from yearn.helpers import s3, telegram logger = logging.getLogger(__name__) sentry_sdk.set_tag('script','curve_apy_previews') @@ -33,7 +27,7 @@ def main(): gauges = _get_gauges() data = _build_data(gauges) - _upload(data) + s3.upload('apy-previews', 'curve-factory', data) def _build_data(gauges): samples = get_samples() @@ -142,99 +136,8 @@ def _get_gauges(): raise ValueError(f"Error fetching gauges from {url}") attempts += 1 sleep(.1) - - else: raise ValueError(f"can't get curve gauges for unsupported network: {chain.id}") - - -def _upload(data): - print(json.dumps(data, sort_keys=True, indent=4)) - - file_name, s3_path = _get_export_paths("curve-factory") - with open(file_name, "w+") as f: - json.dump(data, f) - - if os.getenv("DEBUG", None): - return - - for item in _get_s3s(): - s3 = item["s3"] - aws_bucket = item["aws_bucket"] - s3.upload_file( - file_name, - aws_bucket, - s3_path, - ExtraArgs={'ContentType': "application/json", 'CacheControl': "max-age=1800"}, - ) - - -def _get_s3s(): - s3s = [] - aws_buckets = os.environ.get("AWS_BUCKET").split(";") - aws_endpoint_urls = os.environ.get("AWS_ENDPOINT_URL").split(";") - aws_keys = os.environ.get("AWS_ACCESS_KEY").split(";") - aws_secrets = os.environ.get("AWS_ACCESS_SECRET").split(";") - - for i in range(len(aws_buckets)): - aws_bucket = aws_buckets[i] - aws_endpoint_url = aws_endpoint_urls[i] - aws_key = aws_keys[i] - aws_secret = aws_secrets[i] - kwargs = {} - if aws_endpoint_url is not None: - kwargs["endpoint_url"] = aws_endpoint_url - if aws_key is not None: - kwargs["aws_access_key_id"] = aws_key - if aws_secret is not None: - kwargs["aws_secret_access_key"] = aws_secret - - s3s.append( - { - "s3": boto3.client("s3", **kwargs), - "aws_bucket": aws_bucket - } - ) - - return s3s - - -def _get_export_paths(suffix): - out = "generated" - if os.path.isdir(out): - shutil.rmtree(out) - os.makedirs(out, exist_ok=True) - - api_path = os.path.join("v1", "chains", f"{chain.id}", "apy-previews") - - file_base_path = os.path.join(out, api_path) - os.makedirs(file_base_path, exist_ok=True) - - file_name = os.path.join(file_base_path, suffix) - s3_path = os.path.join(api_path, suffix) - return file_name, s3_path - + def with_monitoring(): - if os.getenv("DEBUG", None): - main() - return - from telegram.ext import Updater - - private_group = os.environ.get('TG_YFIREBOT_GROUP_INTERNAL') - public_group = os.environ.get('TG_YFIREBOT_GROUP_EXTERNAL') - updater = Updater(os.environ.get('TG_YFIREBOT')) - now = datetime.now() - message = f"`[{now}]`\nāš™ļø Curve Previews API for {Network.name()} is updating..." - ping = updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown") - ping = ping.message_id - try: - main() - except Exception as error: - tb = traceback.format_exc() - now = datetime.now() - message = f"`[{now}]`\nšŸ”„ Curve Previews API update for {Network.name()} failed!\n```\n{tb}\n```"[:4000] - updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown", reply_to_message_id=ping) - updater.bot.send_message(chat_id=public_group, text=message, parse_mode="Markdown") - raise error - message = f"āœ… Curve Previews API update for {Network.name()} successful!" - updater.bot.send_message(chat_id=private_group, text=message, reply_to_message_id=ping) \ No newline at end of file + telegram.run_job_with_monitoring('Curve Previews API', main) diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py index d36aae65c..375703f0e 100644 --- a/scripts/drome_apy_previews.py +++ b/scripts/drome_apy_previews.py @@ -5,23 +5,18 @@ import asyncio import dataclasses -import json import logging import os -import shutil -import traceback -from datetime import datetime from pprint import pformat from time import time from typing import List, Optional -import boto3 import sentry_sdk from brownie import ZERO_ADDRESS, chain from msgspec import Struct from multicall.utils import await_awaitable from tqdm.asyncio import tqdm_asyncio -from y import ERC20, Contract, Network, magic +from y import Contract, Network, magic from y.exceptions import ContractNotVerified from y.time import get_block_timestamp_async @@ -29,8 +24,9 @@ from yearn.apy.common import SECONDS_PER_YEAR from yearn.apy.curve.simple import Gauge from yearn.apy.velo import COMPOUNDING -from yearn.v2.registry import Registry from yearn.debug import Debug +from yearn.helpers import s3, telegram +from yearn.v2.registry import Registry logger = logging.getLogger(__name__) sentry_sdk.set_tag('script','curve_apy_previews') @@ -38,6 +34,7 @@ class Drome(Struct): """Holds various params for a drome deployment""" label: str + job_name: str sugar: str voter: str # A random vault to check fees @@ -47,12 +44,14 @@ class Drome(Struct): drome = { Network.Optimism: Drome( label='velo', + job_name='Velodrome Previews API', sugar='0x4D996E294B00cE8287C16A2b9A4e637ecA5c939f', voter='0x41c914ee0c7e1a5edcd0295623e6dc557b5abf3c', fee_checker='0xbC61B71562b01a3a4808D3B9291A3Bf743AB3361', ), Network.Base: Drome( label='aero', + job_name='Aerodrome Previews API', sugar='0x2073D8035bB2b0F2e85aAF5a8732C6f397F9ff9b', voter='0x16613524e02ad97eDfeF371bC883F2F5d6C480A5', fee_checker='0xEcFc1e5BDa4d4191c9Cab053ec704347Db87Be5d', @@ -72,7 +71,8 @@ class Drome(Struct): fees = ApyFees(performance=performance_fee, management=management_fee, keep_velo=keep) def main(): - _upload(await_awaitable(_build_data())) + data = await_awaitable(_build_data()) + s3.upload('apy-previews', f'{drome.label}-factory', data) async def _build_data(): start = int(time()) @@ -138,8 +138,7 @@ async def _load_gauge(lp: dict, block: Optional[int] = None) -> Gauge: return Gauge(lp_address, pool, gauge, weight, inflation_rate, working_supply) async def _staking_apy(lp: dict, staking_rewards: Contract, block: Optional[int]=None) -> float: - - current_time = time() if block is None else await get_block_timestamp_async(block) + query_at_time = time() if block is None else await get_block_timestamp_async(block) reward_token, rate, total_supply, end = await asyncio.gather( staking_rewards.rewardToken.coroutine(block_identifier=block), @@ -150,7 +149,7 @@ async def _staking_apy(lp: dict, staking_rewards: Contract, block: Optional[int] rate *= unkeep - if end < current_time or total_supply == 0 or rate == 0: + if end < query_at_time or total_supply == 0 or rate == 0: return Apy(f"v2:{drome.label}_unpopular", gross_apr=0, net_apy=0, fees=fees) pool_price, token_price = await asyncio.gather( @@ -166,94 +165,5 @@ async def _staking_apy(lp: dict, staking_rewards: Contract, block: Optional[int] logger.info(pformat(Debug().collect_variables(locals()))) return Apy(f"v2:{drome.label}", gross_apr=gross_apr, net_apy=net_apy, fees=fees) -def _upload(data): - print(json.dumps(data, sort_keys=True, indent=4)) - - file_name, s3_path = _get_export_paths("curve-factory") - with open(file_name, "w+") as f: - json.dump(data, f) - - if os.getenv("DEBUG", None): - return - - aws_bucket = os.environ.get("AWS_BUCKET") - - s3 = _get_s3() - s3.upload_file( - file_name, - aws_bucket, - s3_path, - ExtraArgs={'ContentType': "application/json", 'CacheControl': "max-age=1800"}, - ) - - -def _get_s3(): - aws_key = os.environ.get("AWS_ACCESS_KEY") - aws_secret = os.environ.get("AWS_ACCESS_SECRET") - - kwargs = {} - if aws_key is not None: - kwargs["aws_access_key_id"] = aws_key - if aws_secret is not None: - kwargs["aws_secret_access_key"] = aws_secret - - return boto3.client("s3", **kwargs) - - -def _get_export_paths(suffix): - out = "generated" - if os.path.isdir(out): - shutil.rmtree(out) - os.makedirs(out, exist_ok=True) - - api_path = os.path.join("v1", "chains", f"{chain.id}", "apy-previews") - - file_base_path = os.path.join(out, api_path) - os.makedirs(file_base_path, exist_ok=True) - - file_name = os.path.join(file_base_path, suffix) - s3_path = os.path.join(api_path, suffix) - return file_name, s3_path - def with_monitoring(): - if os.getenv("DEBUG", None): - main() - return - from telegram.ext import Updater - - private_group = os.environ.get('TG_YFIREBOT_GROUP_INTERNAL') - public_group = os.environ.get('TG_YFIREBOT_GROUP_EXTERNAL') - updater = Updater(os.environ.get('TG_YFIREBOT')) - now = datetime.now() - if Network.name() == "Optimism": - message = f"`[{now}]`\nāš™ļø Velodrome Previews API for {Network.name()} is updating..." - ping = updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown") - ping = ping.message_id - try: - main() - except Exception as error: - tb = traceback.format_exc() - now = datetime.now() - message = f"`[{now}]`\nšŸ”„ Velodrome Previews API update for {Network.name()} failed!\n```\n{tb}\n```"[:4000] - updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown", reply_to_message_id=ping) - updater.bot.send_message(chat_id=public_group, text=message, parse_mode="Markdown") - raise error - message = f"āœ… Velodrome Previews API update for {Network.name()} successful!" - updater.bot.send_message(chat_id=private_group, text=message, reply_to_message_id=ping) - elif Network.name() == "Base": - message = f"`[{now}]`\nāš™ļø Aerodrome Previews API for {Network.name()} is updating..." - ping = updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown") - ping = ping.message_id - try: - main() - except Exception as error: - tb = traceback.format_exc() - now = datetime.now() - message = f"`[{now}]`\nšŸ”„ Aerodrome Previews API update for {Network.name()} failed!\n```\n{tb}\n```"[:4000] - updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown", reply_to_message_id=ping) - updater.bot.send_message(chat_id=public_group, text=message, parse_mode="Markdown") - raise error - message = f"āœ… Aerodrome Previews API update for {Network.name()} successful!" - updater.bot.send_message(chat_id=private_group, text=message, reply_to_message_id=ping) - else: - message = f"{Network.name()} network not a valid network for previews script." \ No newline at end of file + telegram.run_job_with_monitoring(drome.job_name, main) diff --git a/yearn/helpers/s3.py b/yearn/helpers/s3.py new file mode 100644 index 000000000..30b84bf81 --- /dev/null +++ b/yearn/helpers/s3.py @@ -0,0 +1,68 @@ +import os +import shutil, json +from typing import List, TypedDict, Any + +import boto3 +from brownie import chain + +print(boto3.__dict__) + +class S3(TypedDict): + s3: boto3.client + aws_bucket: str + +def get_s3s() -> List[S3]: + s3s = [] + aws_buckets = os.environ.get("AWS_BUCKET").split(";") + aws_endpoint_urls = os.environ.get("AWS_ENDPOINT_URL").split(";") + aws_keys = os.environ.get("AWS_ACCESS_KEY").split(";") + aws_secrets = os.environ.get("AWS_ACCESS_SECRET").split(";") + + for i in range(len(aws_buckets)): + aws_bucket = aws_buckets[i] + aws_endpoint_url = aws_endpoint_urls[i] + aws_key = aws_keys[i] + aws_secret = aws_secrets[i] + kwargs = {} + if aws_endpoint_url is not None: + kwargs["endpoint_url"] = aws_endpoint_url + if aws_key is not None: + kwargs["aws_access_key_id"] = aws_key + if aws_secret is not None: + kwargs["aws_secret_access_key"] = aws_secret + + s3s.append(S3(s3=boto3.client("s3", **kwargs), aws_bucket=aws_bucket)) + return s3s + +def get_export_paths(path_presufix: str, path_suffix: str): + out = "generated" + if os.path.isdir(out): + shutil.rmtree(out) + os.makedirs(out, exist_ok=True) + + api_path = os.path.join("v1", "chains", f"{chain.id}", path_presufix) + + file_base_path = os.path.join(out, api_path) + os.makedirs(file_base_path, exist_ok=True) + + file_name = os.path.join(file_base_path, path_suffix) + s3_path = os.path.join(api_path, path_suffix) + return file_name, s3_path + +def upload(path_presufix: str, path_suffix: str, data: Any) -> None: + print(json.dumps(data, sort_keys=True, indent=4)) + + file_name, s3_path = get_export_paths(path_presufix, path_suffix) + with open(file_name, "w+") as f: + json.dump(data, f) + + if os.getenv("DEBUG", None): + return + + for s3 in get_s3s(): + s3["s3"].upload_file( + file_name, + s3["aws_bucket"], + s3_path, + ExtraArgs={'ContentType': "application/json", 'CacheControl': "max-age=1800"}, + ) \ No newline at end of file diff --git a/yearn/helpers/telegram.py b/yearn/helpers/telegram.py new file mode 100644 index 000000000..6ada71f3f --- /dev/null +++ b/yearn/helpers/telegram.py @@ -0,0 +1,39 @@ + +import os +import traceback +from datetime import datetime +from typing import Callable, TypeVar + +from y import Network + +T = TypeVar('T') + +PRIVATE_GROUP = os.environ.get('TG_YFIREBOT_GROUP_INTERNAL') +PUBLIC_GROUP = os.environ.get('TG_YFIREBOT_GROUP_EXTERNAL') + + +def run_job_with_monitoring(job_name: str, job: Callable[[], T]) -> T: + """A helper function used when we want to run a job and monitor it via telegram""" + + if os.getenv("DEBUG", None): + return job() + + from telegram.ext import Updater + UPDATER = Updater(os.environ.get('TG_YFIREBOT')) + + now = datetime.now() + message = f"`[{now}]`\nāš™ļø {job_name} for {Network.name()} is updating..." + ping = UPDATER.bot.send_message(chat_id=PRIVATE_GROUP, text=message, parse_mode="Markdown") + ping = ping.message_id + try: + retval = job() + except Exception as error: + tb = traceback.format_exc() + now = datetime.now() + message = f"`[{now}]`\nšŸ”„ {job_name} update for {Network.name()} failed!\n```\n{tb}\n```"[:4000] + UPDATER.bot.send_message(chat_id=PRIVATE_GROUP, text=message, parse_mode="Markdown", reply_to_message_id=ping) + UPDATER.bot.send_message(chat_id=PUBLIC_GROUP, text=message, parse_mode="Markdown") + raise error + message = f"āœ… {job_name} update for {Network.name()} successful!" + UPDATER.bot.send_message(chat_id=PRIVATE_GROUP, text=message, reply_to_message_id=ping) + return retval \ No newline at end of file From 5da48da88243c528e94edb505669c82b28563ae5 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 19/59] fix: BatchSizeError on base --- scripts/exporters/transactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exporters/transactions.py b/scripts/exporters/transactions.py index 7483eb0db..23752d034 100644 --- a/scripts/exporters/transactions.py +++ b/scripts/exporters/transactions.py @@ -39,7 +39,7 @@ Network.Gnosis: 2_000_000, Network.Arbitrum: 1_500_000, Network.Optimism: 4_000_000, - Network.Base: 100_000, + Network.Base: 500_000, }[chain.id] FIRST_END_BLOCK = { From 0fd85a64831522ded7d5b0245973b19ff202ba11 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 20/59] feat: filter 0 value transfers from treasury export --- scripts/exporters/treasury_transactions.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/exporters/treasury_transactions.py b/scripts/exporters/treasury_transactions.py index 2f51e51cc..011896a6a 100644 --- a/scripts/exporters/treasury_transactions.py +++ b/scripts/exporters/treasury_transactions.py @@ -53,9 +53,11 @@ def main() -> NoReturn: @a_sync(default='sync') async def load_new_txs(start_block: Block, end_block: Block) -> int: - futs = [] - async for entry in treasury.ledger._get_and_yield(start_block, end_block): - futs.append(asyncio.create_task(insert_treasury_tx(entry))) + futs = [ + asyncio.create_task(insert_treasury_tx(entry)) + async for entry in treasury.ledger._get_and_yield(start_block, end_block) + if entry.value + ] return sum(await tqdm_asyncio.gather(*futs, desc="Insert Txs to Postgres")) From a50ae78bd9835e194242d3c50f7f3c46ec444b68 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 21/59] fix: BadRequest tg err --- scripts/s3.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/s3.py b/scripts/s3.py index f77b953e3..fcc2c9993 100644 --- a/scripts/s3.py +++ b/scripts/s3.py @@ -345,15 +345,10 @@ def with_monitoring(): tb = traceback.format_exc() now = datetime.now() message = f"`[{now}]`\nšŸ”„ {export_mode} Vaults API update for {Network.name()} failed!\n" - try: + with suppress(BadRequest): detail_message = (message + f"```\n{tb}\n```")[:4000] updater.bot.send_message(chat_id=private_group, text=detail_message, parse_mode="Markdown", reply_to_message_id=ping) updater.bot.send_message(chat_id=public_group, text=detail_message, parse_mode="Markdown") - except BadRequest: - pass - #detail_message = message + f"{error.__class__.__name__}({error})" - #updater.bot.send_message(chat_id=private_group, text=detail_message, parse_mode="Markdown", reply_to_message_id=ping) - #updater.bot.send_message(chat_id=public_group, text=detail_message, parse_mode="Markdown") raise error message = f"āœ… {export_mode} Vaults API update for {Network.name()} successful!" updater.bot.send_message(chat_id=private_group, text=message, reply_to_message_id=ping) From b774febb053db3a7404791ebbefb2f382caaef16 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 22/59] feat: async log loading --- scripts/drome_apy_previews.py | 2 +- scripts/exporters/wallets.py | 2 +- scripts/historical_tvl.py | 2 +- scripts/print_strategies.py | 4 +- scripts/s3.py | 14 +- scripts/tokenlist.py | 2 +- scripts/tvl.py | 2 +- yearn/apy/aero.py | 5 +- yearn/apy/curve/simple.py | 26 ++-- yearn/apy/v2.py | 14 +- yearn/apy/velo.py | 5 +- yearn/decorators.py | 31 +++- yearn/events.py | 2 +- yearn/iearn.py | 2 +- yearn/ironbank.py | 4 +- yearn/special.py | 9 +- yearn/v1/registry.py | 6 +- yearn/v1/vaults.py | 15 +- yearn/v2/registry.py | 264 ++++++++++++++++++++++------------ yearn/v2/strategies.py | 130 +++++++---------- yearn/yearn.py | 26 ++-- 21 files changed, 317 insertions(+), 250 deletions(-) diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py index 375703f0e..b89f56f52 100644 --- a/scripts/drome_apy_previews.py +++ b/scripts/drome_apy_previews.py @@ -85,7 +85,7 @@ async def _build_data(): async def _get_lps_with_vault_potential() -> List[dict]: sugar_oracle = await Contract.coroutine(drome.sugar) - current_vaults = Registry(watch_events_forever=False, include_experimental=False).vaults + current_vaults = Registry(include_experimental=False).vaults current_underlyings = [str(vault.token) for vault in current_vaults] return [lp for lp in await sugar_oracle.all.coroutine(999999999999999999999, 0, ZERO_ADDRESS) if lp[0] not in current_underlyings and lp[11] != ZERO_ADDRESS] diff --git a/scripts/exporters/wallets.py b/scripts/exporters/wallets.py index 729d39d2c..fd681e015 100644 --- a/scripts/exporters/wallets.py +++ b/scripts/exporters/wallets.py @@ -20,7 +20,7 @@ logger = logging.getLogger('yearn.wallet_exporter') -yearn = Yearn(load_strategies=False, watch_events_forever=False) +yearn = Yearn() # start: 2020-02-12 first iearn deployment # start opti: 2022-01-01 an arbitrary start timestamp because the default start is < block 1 on opti and messes things up diff --git a/scripts/historical_tvl.py b/scripts/historical_tvl.py index d65404746..eedde109b 100644 --- a/scripts/historical_tvl.py +++ b/scripts/historical_tvl.py @@ -29,7 +29,7 @@ def generate_snapshot_range(start, interval): def main(): - yearn = Yearn(load_strategies=False) + yearn = Yearn() start = START_DATE[chain.id] interval = timedelta(hours=24) diff --git a/scripts/print_strategies.py b/scripts/print_strategies.py index 1fd8b20d4..98c535682 100644 --- a/scripts/print_strategies.py +++ b/scripts/print_strategies.py @@ -2,6 +2,7 @@ import click import sentry_sdk +from multicall.utils import await_awaitable from brownie.utils.output import build_tree sentry_sdk.set_tag('script','print_strategies') @@ -11,7 +12,6 @@ def main(): from yearn.v2.registry import Registry registry = Registry() print(registry) - registry.load_strategies() tree = [] for vault in registry.vaults: transforms = { @@ -28,7 +28,7 @@ def main(): 'totalLoss': lambda tokens: f'{tokens / vault.scale}', } strategies = [] - for strategy in vault.strategies + vault.revoked_strategies: + for strategy in await_awaitable(vault.strategies) + await_awaitable(vault.revoked_strategies): config = vault.vault.strategies(strategy.strategy).dict() color = 'green' if strategy in vault.strategies else 'red' strategies.append([ diff --git a/scripts/s3.py b/scripts/s3.py index fcc2c9993..1185d7130 100644 --- a/scripts/s3.py +++ b/scripts/s3.py @@ -7,6 +7,7 @@ import shutil import traceback import warnings +from contextlib import suppress from datetime import datetime from time import time from typing import Union @@ -25,11 +26,9 @@ from yearn import logs from yearn.apy import (Apy, ApyBlocks, ApyFees, ApyPoints, ApySamples, get_samples) -from yearn.common import Tvl from yearn.exceptions import EmptyS3Export from yearn.graphite import send_metric from yearn.special import Backscratcher, YveCRVJar -from yearn.utils import chunks, contract from yearn.v1.registry import Registry as RegistryV1 from yearn.v1.vaults import VaultV1 from yearn.v2.registry import Registry as RegistryV2 @@ -62,7 +61,8 @@ async def wrap_vault( } ] else: - strategies = [{"address": str(strategy.strategy), "name": strategy.name} for strategy in vault.strategies] + strategies = await vault.strategies if isinstance(vault, VaultV2) else vault.strategies + strategies = [{"address": str(strategy.strategy), "name": strategy.name} for strategy in strategies] token_alias = aliases[str(vault.token)]["symbol"] if str(vault.token) in aliases else await ERC20(vault.token, asynchronous=True).symbol vault_alias = token_alias @@ -90,7 +90,7 @@ async def wrap_vault( "tvl": dataclasses.asdict(await tvl_fut), "apy": dataclasses.asdict(await apy_fut), "strategies": strategies, - "endorsed": vault.is_endorsed if hasattr(vault, "is_endorsed") else True, + "endorsed": await vault.is_endorsed if hasattr(vault, "is_endorsed") else True, "version": vault.api_version if hasattr(vault, "api_version") else "0.1", "decimals": vault.decimals if hasattr(vault, "decimals") else await ERC20(vault.vault, asynchronous=True).decimals, "type": "v2" if isinstance(vault, VaultV2) else "v1", @@ -216,14 +216,14 @@ async def _main(): if chain.id == Network.Mainnet: special = [YveCRVJar(), Backscratcher()] registry_v1 = RegistryV1() - vaults = list(itertools.chain(special, registry_v1.vaults, registry_v2.vaults, registry_v2.experiments)) + vaults = list(itertools.chain(special, registry_v1.vaults, await registry_v2.vaults, await registry_v2.experiments)) else: - vaults = registry_v2.vaults + vaults = await registry_v2.vaults if len(vaults) == 0: raise ValueError(f"No vaults found for chain_id: {chain.id}") - assets_metadata = await get_assets_metadata(registry_v2.vaults) + assets_metadata = await get_assets_metadata(await registry_v2.vaults) data = [] total = len(vaults) diff --git a/scripts/tokenlist.py b/scripts/tokenlist.py index 50456750c..e566238b8 100644 --- a/scripts/tokenlist.py +++ b/scripts/tokenlist.py @@ -18,7 +18,7 @@ def main(): - yearn = Yearn(load_strategies=False) + yearn = Yearn() excluded = { "0xBa37B002AbaFDd8E89a1995dA52740bbC013D992", "0xe2F6b9773BF3A015E2aA70741Bde1498bdB9425b", diff --git a/scripts/tvl.py b/scripts/tvl.py index 928f17973..c8c7e8fd9 100644 --- a/scripts/tvl.py +++ b/scripts/tvl.py @@ -14,7 +14,7 @@ def main(): data = [] - yearn = Yearn(load_strategies=False) + yearn = Yearn() for product, registry in yearn.registries.items(): for name, tvl in registry.total_value_at().items(): diff --git a/yearn/apy/aero.py b/yearn/apy/aero.py index 653cc5fcf..cce5e30d0 100644 --- a/yearn/apy/aero.py +++ b/yearn/apy/aero.py @@ -28,7 +28,7 @@ async def get_staking_pool(underlying: str) -> Optional[Contract]: return None if staking_pool == ZERO_ADDRESS else await Contract.coroutine(staking_pool) async def staking(vault: "Vault", staking_rewards: Contract, samples: ApySamples, block: Optional[int]=None) -> float: - if len(vault.strategies) == 0: + if len(await vault.strategies) == 0: return Apy("v2:aero_no_strats", 0, 0, ApyFees(0, 0), ApyPoints(0, 0, 0)) end = await staking_rewards.periodFinish.coroutine(block_identifier=block) @@ -38,7 +38,8 @@ async def staking(vault: "Vault", staking_rewards: Contract, samples: ApySamples performance = await vault.vault.performanceFee.coroutine(block_identifier=block) / 1e4 if hasattr(vault.vault, "performanceFee") else 0 management = await vault.vault.managementFee.coroutine(block_identifier=block) / 1e4 if hasattr(vault.vault, "managementFee") else 0 # since its a fork we still call it keepVELO - keep = await vault.strategies[0].strategy.localKeepVELO.coroutine(block_identifier=block) / 1e4 if hasattr(vault.strategies[0].strategy, "localKeepVELO") else 0 + strats = await vault.strategies + keep = await strats[0].strategy.localKeepVELO.coroutine(block_identifier=block) / 1e4 if hasattr(strats[0].strategy, "localKeepVELO") else 0 rate = rate * (1 - keep) fees = ApyFees(performance=performance, management=management, keep_velo=keep) diff --git a/yearn/apy/curve/simple.py b/yearn/apy/curve/simple.py index c8ce23d0c..5093a7b64 100644 --- a/yearn/apy/curve/simple.py +++ b/yearn/apy/curve/simple.py @@ -272,10 +272,11 @@ async def calculate_simple(vault, gauge: Gauge, samples: ApySamples) -> Apy: if vault: if isinstance(vault, VaultV2): vault_contract = vault.vault - if len(vault.strategies) > 0 and hasattr(vault.strategies[0].strategy, "keepCRV"): - crv_keep_crv = await vault.strategies[0].strategy.keepCRV.coroutine(block_identifier=block) / 1e4 - elif len(vault.strategies) > 0 and hasattr(vault.strategies[0].strategy, "keepCrvPercent"): - crv_keep_crv = await vault.strategies[0].strategy.keepCrvPercent.coroutine(block_identifier=block) / 1e4 + strats = await vault.strategies + if len(strats) > 0 and hasattr(strats[0].strategy, "keepCRV"): + crv_keep_crv = await strats[0].strategy.keepCRV.coroutine(block_identifier=block) / 1e4 + elif len(strats) > 0 and hasattr(strats[0].strategy, "keepCrvPercent"): + crv_keep_crv = await strats[0].strategy.keepCrvPercent.coroutine(block_identifier=block) / 1e4 else: crv_keep_crv = 0 performance = await vault_contract.performanceFee.coroutine(block_identifier=block) / 1e4 if hasattr(vault_contract, "performanceFee") else 0 @@ -298,24 +299,25 @@ async def calculate_simple(vault, gauge: Gauge, samples: ApySamples) -> Apy: cvx_vault = None # if the vault consists of only a convex strategy then return # specialized apy calculations for convex - if _ConvexVault.is_convex_vault(vault): - cvx_strategy = vault.strategies[0].strategy + if await _ConvexVault.is_convex_vault(vault): + strats = await vault.strategies + cvx_strategy = strats[0].strategy cvx_vault = _ConvexVault(cvx_strategy, vault, gauge.gauge) return await cvx_vault.apy(base_asset_price, pool_price, base_apr, pool_apy, management, performance) # if the vault has two strategies then the first is curve and the second is convex - if isinstance(vault, VaultV2) and len(vault.strategies) == 2: # this vault has curve and convex + if isinstance(vault, VaultV2) and len(strats := await vault.strategies) == 2: # this vault has curve and convex # The first strategy should be curve, the second should be convex. # However the order on the vault object here does not correspond # to the order on the withdrawal queue on chain, therefore we need to # re-order so convex is always second if necessary - first_strategy = vault.strategies[0].strategy - second_strategy = vault.strategies[1].strategy + first_strategy = strats[0].strategy + second_strategy = strats[1].strategy crv_strategy = first_strategy cvx_strategy = second_strategy - if not _ConvexVault.is_convex_strategy(vault.strategies[1]): + if not _ConvexVault.is_convex_strategy(strats[1]): cvx_strategy = first_strategy crv_strategy = second_strategy @@ -383,7 +385,7 @@ def __init__(self, cvx_strategy, vault, gauge, block=None) -> None: self.gauge = gauge @staticmethod - def is_convex_vault(vault) -> bool: + async def is_convex_vault(vault) -> bool: """Determines whether the passed in vault is a Convex vault i.e. it only has one strategy that's based on farming Convex. """ @@ -392,7 +394,7 @@ def is_convex_vault(vault) -> bool: if not isinstance(vault, VaultV2): return False - return len(vault.strategies) == 1 and _ConvexVault.is_convex_strategy(vault.strategies[0]) + return len(strats := await vault.strategies) == 1 and _ConvexVault.is_convex_strategy(strats[0]) @staticmethod def is_convex_strategy(strategy) -> bool: diff --git a/yearn/apy/v2.py b/yearn/apy/v2.py index 8d491c5eb..25cf1f7d7 100644 --- a/yearn/apy/v2.py +++ b/yearn/apy/v2.py @@ -6,13 +6,12 @@ from brownie import chain from semantic_version.base import Version -from y.networks import Network +from y import Network from yearn.apy.common import (Apy, ApyBlocks, ApyError, ApyFees, ApyPoints, ApySamples, SharePricePoint, calculate_roi) from yearn.apy.staking_rewards import get_staking_rewards_apr from yearn.debug import Debug -from yearn.utils import run_in_thread logger = logging.getLogger(__name__) @@ -29,9 +28,8 @@ def closest(haystack, needle): else: return before - async def simple(vault, samples: ApySamples) -> Apy: - harvests = sorted([harvest for strategy in vault.strategies for harvest in await run_in_thread(getattr, strategy, "harvests")]) + harvests = sorted([harvest for strategy in await vault.strategies async for harvest in strategy.harvests(samples.now)]) # we don't want to display APYs when vaults are ramping up if len(harvests) < 2: @@ -51,7 +49,7 @@ async def simple(vault, samples: ApySamples) -> Apy: # get our inception data # the first report is when the vault first allocates funds to farm with - reports = await run_in_thread(getattr, vault, 'reports') + reports = await vault.reports inception_block = reports[0].block_number inception_price = await price_per_share(block_identifier=inception_block) @@ -91,7 +89,7 @@ async def simple(vault, samples: ApySamples) -> Apy: # for performance fee, half comes from strategy (strategist share) and half from the vault (treasury share) strategy_fees = [] - for strategy in vault.strategies: # look at all of our strategies + for strategy in await vault.strategies: # look at all of our strategies strategy_info = await contract.strategies.coroutine(strategy.strategy) debt_ratio = strategy_info['debtRatio'] / 10000 performance_fee = strategy_info['performanceFee'] @@ -126,7 +124,7 @@ async def simple(vault, samples: ApySamples) -> Apy: async def average(vault, samples: ApySamples) -> Apy: - reports = await run_in_thread(getattr, vault, "reports") + reports = await vault.reports # we don't want to display APYs when vaults are ramping up if len(reports) < 2: @@ -189,7 +187,7 @@ async def average(vault, samples: ApySamples) -> Apy: # for performance fee, half comes from strategy (strategist share) and half from the vault (treasury share) strategy_fees = [] - for strategy in vault.strategies: # look at all of our strategies + for strategy in await vault.strategies: # look at all of our strategies strategy_info = await contract.strategies.coroutine(strategy.strategy) debt_ratio = strategy_info['debtRatio'] / 10000 performance_fee = strategy_info['performanceFee'] diff --git a/yearn/apy/velo.py b/yearn/apy/velo.py index 4afd16e59..89892ff32 100644 --- a/yearn/apy/velo.py +++ b/yearn/apy/velo.py @@ -28,7 +28,7 @@ async def get_staking_pool(underlying: str) -> Optional[Contract]: return None if staking_pool == ZERO_ADDRESS else await Contract.coroutine(staking_pool) async def staking(vault: "Vault", staking_rewards: Contract, samples: ApySamples, block: Optional[int]=None) -> float: - if len(vault.strategies) == 0: + if len(await vault.strategies) == 0: return Apy("v2:velo_no_strats", 0, 0, ApyFees(0, 0), ApyPoints(0, 0, 0)) end = await staking_rewards.periodFinish.coroutine(block_identifier=block) @@ -37,7 +37,8 @@ async def staking(vault: "Vault", staking_rewards: Contract, samples: ApySamples rate = await staking_rewards.rewardRate.coroutine(block_identifier=block) if hasattr(staking_rewards, "rewardRate") else 0 performance = await vault.vault.performanceFee.coroutine(block_identifier=block) / 1e4 if hasattr(vault.vault, "performanceFee") else 0 management = await vault.vault.managementFee.coroutine(block_identifier=block) / 1e4 if hasattr(vault.vault, "managementFee") else 0 - keep = await vault.strategies[0].strategy.localKeepVELO.coroutine(block_identifier=block) / 1e4 if hasattr(vault.strategies[0].strategy, "localKeepVELO") else 0 + strats = vault.strategies + keep = await strats[0].strategy.localKeepVELO.coroutine(block_identifier=block) / 1e4 if hasattr(strats[0].strategy, "localKeepVELO") else 0 rate = rate * (1 - keep) fees = ApyFees(performance=performance, management=management, keep_velo=keep) diff --git a/yearn/decorators.py b/yearn/decorators.py index 3604394ae..c1f1bd198 100644 --- a/yearn/decorators.py +++ b/yearn/decorators.py @@ -1,5 +1,5 @@ import _thread -import signal +import asyncio import functools import logging import signal @@ -24,14 +24,31 @@ def wrap(self): def wait_or_exit_before(func): @functools.wraps(func) - def wrap(self): - self._done.wait() - if self._has_exception: - logger.error(self._exception) - _thread.interrupt_main(signal.SIGTERM) - return func(self) + async def wrap(self): + task: asyncio.Task = self._task + logger.info("waiting for %s", self) + while not self._done.is_set() and not task.done(): + await asyncio.sleep(10) + logger.info("%s not done", self) + logger.info("loading %s complete", self) + if task.done() and (e := task.exception()): + logger.info('task %s has exception %s, awaiting', task, e) + raise e + return await func(self) return wrap +_main_thread_loop = asyncio.get_event_loop() + +def set_exc(func): + @functools.wraps(func) + async def wrap(self): + # in case this loads in a diff thread + try: + return await func(self) + except Exception as e: + self._done.set() + raise e + return wrap def wait_or_exit_after(func): @functools.wraps(func) diff --git a/yearn/events.py b/yearn/events.py index 13d0967c8..ed471d34e 100644 --- a/yearn/events.py +++ b/yearn/events.py @@ -68,7 +68,7 @@ def get_logs_asap( if verbose > 0: logger.info('fetching %d batches', len(ranges)) - batches = Parallel(8, "threading", verbose=verbose)( + batches = Parallel(64, "threading", verbose=verbose)( delayed(web3.eth.get_logs)(_get_logs_params(addresses, topics, start, end)) for start, end in ranges ) diff --git a/yearn/iearn.py b/yearn/iearn.py index 495f090bb..2996b9a9e 100644 --- a/yearn/iearn.py +++ b/yearn/iearn.py @@ -66,7 +66,7 @@ async def describe(self, block=None) -> dict: "pooled balance": res["pool"] / vault.scale, "price per share": res['getPricePerFullShare'] / 1e18, "token price": price, - "tvl": res["pool"] / vault.scale * price, + "tvl": res["pool"] / vault.scale * float(price), "address": vault.vault, "version": "iearn", } diff --git a/yearn/ironbank.py b/yearn/ironbank.py index 65c9deba1..f7cf620f4 100644 --- a/yearn/ironbank.py +++ b/yearn/ironbank.py @@ -110,7 +110,7 @@ async def describe(self, block=None): for attr in ["getCash", "totalBorrows", "totalReserves"]: res[attr] /= 10 ** m.decimals - tvl = (res["getCash"] + res["totalBorrows"] - res["totalReserves"]) * price + tvl = (res["getCash"] + res["totalBorrows"] - res["totalReserves"]) * float(price) supplied = res["getCash"] + res["totalBorrows"] - res["totalReserves"] ratio = res["totalBorrows"] / supplied if supplied != 0 else None @@ -121,7 +121,7 @@ async def describe(self, block=None): "total borrows": res["totalBorrows"], "total reserves": res["totalReserves"], "exchange rate": exchange_rate, - "token price": price * exchange_rate, + "token price": float(price) * exchange_rate, "underlying price": price, "supply apy": res["supplyRatePerBlock"] / 1e18 * blocks_per_year, "borrow apy": res["borrowRatePerBlock"] / 1e18 * blocks_per_year, diff --git a/yearn/special.py b/yearn/special.py index e77275caf..8c79a560f 100644 --- a/yearn/special.py +++ b/yearn/special.py @@ -6,7 +6,7 @@ import eth_retry import requests from brownie import chain -from y import Contract, magic +from y import ERC20, Contract, magic from y.contracts import contract_creation_block_async from y.exceptions import PriceError, yPriceMagicError @@ -83,7 +83,7 @@ async def describe(self, block=None): return { 'totalSupply': crv_locked, 'token price': crv_price, - 'tvl': crv_locked * crv_price, + 'tvl': crv_locked * float(crv_price), } async def total_value_at(self, block=None): @@ -136,12 +136,9 @@ async def tvl(self, block=None) -> Tvl: if not isinstance(e.exception, PriceError): raise e price = None - tvl = total_assets * price / 10 ** await self.vault.decimals.coroutine(block_identifier=block) if price else None + tvl = total_assets * price / await ERC20(self.vault, asynchronous=True).scale if price else None return Tvl(total_assets, price, tvl) - - - class Ygov(metaclass = Singleton): def __init__(self): self.name = "yGov" diff --git a/yearn/v1/registry.py b/yearn/v1/registry.py index 41963e170..0a752eb28 100644 --- a/yearn/v1/registry.py +++ b/yearn/v1/registry.py @@ -3,9 +3,10 @@ from functools import cached_property from typing import Dict, List, Optional -from brownie import chain, interface, web3 +from brownie import chain, interface from dank_mids.brownie_patch import patch_contract from y.contracts import contract_creation_block_async +from y.decorators import stuck_coro_debugger from y.networks import Network from y.utils.dank_mids import dank_w3 @@ -37,6 +38,7 @@ def vaults(self) -> List[VaultV1]: def __repr__(self) -> str: return f"" + @stuck_coro_debugger async def describe(self, block: Optional[Block] = None) -> Dict[str, Dict]: vaults = await self.active_vaults_at(block) share_prices = await fetch_multicall_async(*[[vault.vault, "getPricePerFullShare"] for vault in vaults], block=block) @@ -44,6 +46,7 @@ async def describe(self, block: Optional[Block] = None) -> Dict[str, Dict]: data = await asyncio.gather(*[vault.describe(block=block) for vault in vaults]) return {vault.name: desc for vault, desc in zip(vaults, data)} + @stuck_coro_debugger async def total_value_at(self, block: Optional[Block] = None) -> Dict[str, float]: vaults = await self.active_vaults_at(block) balances = await fetch_multicall_async(*[[vault.vault, "balance"] for vault in vaults], block=block) @@ -52,6 +55,7 @@ async def total_value_at(self, block: Optional[Block] = None) -> Dict[str, float prices = await asyncio.gather(*[vault.get_price(block) for (vault, balance) in vaults]) return {vault.name: balance * price / 10 ** vault.decimals for (vault, balance), price in zip(vaults, prices)} + @stuck_coro_debugger async def active_vaults_at(self, block: Optional[Block] = None) -> List[VaultV1]: if block: blocks = await asyncio.gather(*[contract_creation_block_async(str(vault.vault)) for vault in self.vaults]) diff --git a/yearn/v1/vaults.py b/yearn/v1/vaults.py index c51b688fb..9bcbb8620 100644 --- a/yearn/v1/vaults.py +++ b/yearn/v1/vaults.py @@ -7,8 +7,9 @@ from brownie import ZERO_ADDRESS, interface from brownie.network.contract import InterfaceContainer from dank_mids.brownie_patch import patch_contract +from y import Contract, magic +from y.decorators import stuck_coro_debugger from y.exceptions import PriceError, yPriceMagicError -from y.prices import magic from y.utils.dank_mids import dank_w3 from yearn import constants @@ -53,6 +54,7 @@ async def get_price(self, block=None): return await magic.get_price(underlying, block=block, sync=False) return await magic.get_price(self.token, block=block, sync=False) + @stuck_coro_debugger async def get_strategy(self, block=None): if self.name in ["aLINK", "LINK"] or block is None: return self.strategy @@ -62,15 +64,18 @@ async def get_strategy(self, block=None): if strategy != ZERO_ADDRESS: return contract(strategy) + @stuck_coro_debugger async def get_controller(self, block=None): if block is None: return self.controller - return contract(self.vault.controller(block_identifier=block)) + return await Contract.coroutine(await self.vault.controller.coroutine(block_identifier=block)) @async_cached_property + @stuck_coro_debugger async def is_curve_vault(self): return await magic.curve.get_pool(str(self.token)) is not None + @stuck_coro_debugger async def describe(self, block=None): info = {} strategy = self.strategy @@ -139,18 +144,20 @@ async def describe(self, block=None): if "token price" not in info: info["token price"] = await self.get_price(block=block) if info["vault total"] > 0 else 0 - info["tvl"] = info["vault balance"] * info["token price"] + info["tvl"] = info["vault balance"] * float(info["token price"]) return info + @stuck_coro_debugger async def apy(self, samples: "ApySamples"): from yearn import apy from yearn.prices.curve import curve - if curve.get_pool(self.token.address): + if await magic.curve.get_pool(self.token.address): return await apy.curve.simple(self, samples) else: return await apy.v1.simple(self, samples) + @stuck_coro_debugger async def tvl(self, block=None): total_assets = await self.vault.balance.coroutine(block_identifier=block) try: diff --git a/yearn/v2/registry.py b/yearn/v2/registry.py index 4d8741a1a..5e65251ff 100644 --- a/yearn/v2/registry.py +++ b/yearn/v2/registry.py @@ -1,25 +1,29 @@ import asyncio +import itertools import logging -import threading import time from collections import OrderedDict -from typing import Dict, List +from functools import cached_property +from logging import getLogger +from typing import AsyncIterator, Awaitable, Dict, List, NoReturn, overload +import a_sync import inflection -from brownie import Contract, chain, web3 +from async_property import async_cached_property, async_property +from brownie import chain, web3 +from brownie.network.event import _EventItem from dank_mids.brownie_patch import patch_contract -from joblib import Parallel, delayed from web3._utils.abi import filter_by_name from web3._utils.events import construct_event_topic_set -from y.contracts import contract_creation_block_async +from y import Contract +from y.decorators import stuck_coro_debugger from y.exceptions import NodeNotSynced from y.networks import Network from y.prices import magic from y.utils.dank_mids import dank_w3 +from y.utils.events import Events, ProcessedEvents -from yearn.decorators import (sentry_catch_all, wait_or_exit_after, - wait_or_exit_before) -from yearn.events import decode_logs, get_logs_asap +from yearn.decorators import set_exc, wait_or_exit_before from yearn.exceptions import UnsupportedNetwork from yearn.multicall2 import fetch_multicall_async from yearn.utils import Singleton, contract @@ -44,111 +48,114 @@ ] } +VaultName = str + class Registry(metaclass=Singleton): - def __init__(self, watch_events_forever=True, include_experimental=True): + def __init__(self, include_experimental=True): self.releases = {} # api_version => template - self._vaults = {} # address -> Vault - self._experiments = {} # address => Vault - self._staking_pools = {} # vault address -> staking_pool address self.governance = None self.tags = {} - self._watch_events_forever = watch_events_forever self.include_experimental = include_experimental - self.registries = self.load_registry() - # load registry state in the background - self._done = threading.Event() - self._has_exception = False - self._thread = threading.Thread(target=self.watch_events, daemon=True) - self._thread.start() - - def load_registry(self): + self._done = a_sync.Event(name=f"{self.__module__}.{self.__class__.__name__}._done") + self._registries = [] + self._vaults = {} # address -> Vault + self._experiments = {} # address => Vault + self._staking_pools = {} # vault address -> staking_pool address + + @async_cached_property + @stuck_coro_debugger + async def registries(self) -> List[Contract]: if chain.id == Network.Mainnet: - registries = self.load_from_ens() + registries = await self.load_from_ens() elif chain.id == Network.Gnosis: - registries = [contract('0xe2F12ebBa58CAf63fcFc0e8ab5A61b145bBA3462')] + registries = [await Contract.coroutine('0xe2F12ebBa58CAf63fcFc0e8ab5A61b145bBA3462')] elif chain.id == Network.Fantom: - registries = [contract('0x727fe1759430df13655ddb0731dE0D0FDE929b04')] + registries = [await Contract.coroutine('0x727fe1759430df13655ddb0731dE0D0FDE929b04')] elif chain.id == Network.Arbitrum: - registries = [contract('0x3199437193625DCcD6F9C9e98BDf93582200Eb1f')] + registries = [await Contract.coroutine('0x3199437193625DCcD6F9C9e98BDf93582200Eb1f')] elif chain.id == Network.Optimism: - registries = [ - contract('0x79286Dd38C9017E5423073bAc11F53357Fc5C128'), - contract('0x81291ceb9bB265185A9D07b91B5b50Df94f005BF'), - contract('0x8ED9F6343f057870F1DeF47AaE7CD88dfAA049A8'), # StakingRewardsRegistry - ] + registries = await asyncio.gather(*[ + Contract.coroutine('0x79286Dd38C9017E5423073bAc11F53357Fc5C128'), + Contract.coroutine('0x81291ceb9bB265185A9D07b91B5b50Df94f005BF'), + Contract.coroutine('0x8ED9F6343f057870F1DeF47AaE7CD88dfAA049A8'), # StakingRewardsRegistry + ]) elif chain.id == Network.Base: - registries = [contract('0xF3885eDe00171997BFadAa98E01E167B53a78Ec5')] + registries = [await Contract.coroutine('0xF3885eDe00171997BFadAa98E01E167B53a78Ec5')] else: raise UnsupportedNetwork('yearn v2 is not available on this network') for r in registries[:]: if hasattr(r, 'releaseRegistry') and "ReleaseRegistryUpdated" in r.topics: - logs = get_logs_asap(str(r), [r.topics["ReleaseRegistryUpdated"]]) # Add all past and present Release Registries - for rr in {list(event.values())[0] for event in decode_logs(logs)}: - registries.append(contract(rr)) + events = Events(addresses=r, topics=[r.topics['ReleaseRegistryUpdated']]) + for rr in set(await asyncio.gather(*[ + asyncio.create_task(Contract.coroutine(list(event.values())[0])) + async for event in events.events(to_block=await dank_w3.eth.block_number) + ])): + registries.append(rr) logger.debug("release registry %s found for registry %s", rr, r) + logger.info('registry loaded') + events._task.cancel() return registries - def load_from_ens(self): + @stuck_coro_debugger + async def load_from_ens(self): # track older registries to pull experiments - resolver = contract('0x4976fb03C32e5B8cfe2b6cCB31c09Ba78EBaBa41') + resolver = await Contract.coroutine('0x4976fb03C32e5B8cfe2b6cCB31c09Ba78EBaBa41') topics = construct_event_topic_set( filter_by_name('AddressChanged', resolver.abi)[0], web3.codec, {'node': web3.ens.namehash('v2.registry.ychad.eth')}, ) - events = decode_logs(get_logs_asap(str(resolver), topics)) - registries = [contract(event['newAddress'].hex()) for event in events] + events = Events(addresses=resolver, topics=topics) + registries = [ + asyncio.create_task( + coro=Contract.coroutine(event['newAddress'].hex()), + name=f"load registry {event['newAddress']}", + ) + async for event in events.events(to_block = await dank_w3.eth.block_number) + ] + if registries: + registries = await asyncio.gather(*registries) logger.info('loaded %d registry versions', len(registries)) + events._task.cancel() return registries - @property + @async_property + @stuck_coro_debugger @wait_or_exit_before - def vaults(self) -> List[Vault]: + async def vaults(self) -> List[Vault]: return list(self._vaults.values()) - @property + @async_property + @stuck_coro_debugger @wait_or_exit_before - def experiments(self) -> List[Vault]: + async def experiments(self) -> List[Vault]: return list(self._experiments.values()) - @property + @async_property + @stuck_coro_debugger @wait_or_exit_before - def staking_pools(self) -> Dict: + async def staking_pools(self) -> Dict: return self._staking_pools - @wait_or_exit_before def __repr__(self) -> str: - return f"" - - @wait_or_exit_after - def load_vaults(self): - if not self._thread._started.is_set(): - self._thread.start() - - @sentry_catch_all - def watch_events(self): - start = time.time() - sleep_time = 300 - from_block = None - height = chain.height - while True: - logs = get_logs_asap([str(addr) for addr in self.registries], None, from_block=from_block, to_block=height) - self.process_events(decode_logs(logs)) + return f"" + + @set_exc + async def watch_events(self) -> NoReturn: + start = await dank_w3.eth.block_number + events = await self._events + def done_callback(task: asyncio.Task) -> None: + logger.info("loaded v2 registry in %.3fs", time.time() - start) + self._done.set() + done_task = asyncio.create_task(events._lock.wait_for(start)) + done_task.add_done_callback(done_callback) + async for _ in events: self._filter_vaults() if not self._done.is_set(): self._done.set() logger.info("loaded v2 registry in %.3fs", time.time() - start) - if not self._watch_events_forever: - return - time.sleep(sleep_time) - - # set vars for next loop - from_block = height + 1 - height = chain.height - if height < from_block: - raise NodeNotSynced(f"No new blocks in the past {sleep_time/60} minutes.") def process_events(self, events): temp_rekt_vaults = [] @@ -207,23 +214,16 @@ def vault_from_event(self, event): token=event["token"], api_version=event["api_version"], registry=self, - watch_events_forever=self._watch_events_forever, ) - def load_strategies(self): - # stagger loading strategies to not run out of connections in the pool - vaults = self.vaults + self.experiments - Parallel(1, "threading")(delayed(vault.load_strategies)() for vault in vaults) - - def load_harvests(self): - vaults = self.vaults + self.experiments - Parallel(1, "threading")(delayed(vault.load_harvests)() for vault in vaults) - - async def describe(self, block=None): - vaults = await self.active_vaults_at(block) - results = await asyncio.gather(*[vault.describe(block=block) for vault in vaults]) - return {vault.name: result for vault, result in zip(vaults, results)} + @stuck_coro_debugger + async def describe(self, block=None) -> [VaultName, Dict]: + return await a_sync.gather({ + vault.name: asyncio.create_task(vault.describe(block=block)) + async for vault in self.active_vaults_at(block, iter=True) + }) + @stuck_coro_debugger async def total_value_at(self, block=None): vaults = await self.active_vaults_at(block) prices, results = await asyncio.gather( @@ -232,15 +232,44 @@ async def total_value_at(self, block=None): ) return {vault.name: assets * price / vault.scale for vault, assets, price in zip(vaults, results, prices)} - async def active_vaults_at(self, block=None): - vaults = self.vaults + self.experiments - if block: - blocks = await asyncio.gather(*[contract_creation_block_async(str(vault.vault)) for vault in vaults]) - vaults = [vault for vault, deploy_block in zip(vaults, blocks) if deploy_block <= block] - # fixes edge case: a vault is not necessarily initialized on creation - activations = await fetch_multicall_async(*[[vault.vault, 'activation'] for vault in vaults], block=block) - return [vault for vault, activation in zip(vaults, activations) if activation] - + @overload + def active_vaults_at(self, block=None, iter = False) -> Awaitable[List[Vault]]:... + @overload + def active_vaults_at(self, block=None, iter = True) -> AsyncIterator[Vault]:... + def active_vaults_at(self, block=None, iter: bool = False): + if iter: + return self._active_vaults_at_iter(block=block) + else: + return self._active_vaults_at(block=block) + + @stuck_coro_debugger + async def _active_vaults_at(self, block=None) -> List[Vault]: + self._task + events = await self._events + await events._lock.wait_for(events._init_block) + vaults = list(itertools.chain(self._vaults.values(), self._experiments.values())) + return [vault for vault, active in zip(vaults, await asyncio.gather(*[vault.is_active(block) for vault in vaults])) if active] + + async def _active_vaults_at_iter(self, block=None) -> AsyncIterator[Vault]: + self._task + events = await self._events + await events._lock.wait_for(events._init_block) + vaults = list(itertools.chain(self._vaults.values(), self._experiments.values())) + async def is_active(vault: Vault) -> bool: + return vault, await vault.is_active(block) + for fut in asyncio.as_completed([is_active(vault) for vault in vaults]): + vault, active = await fut + if active: + yield vault + + @async_cached_property + async def _events(self) -> "RegistryEvents": + return RegistryEvents(self, await self.registries) + + @cached_property + def _task(self) -> asyncio.Task: + return asyncio.create_task(self.watch_events()) + def _filter_vaults(self): logger.debug('filtering vaults') if chain.id in DEPRECATED_VAULTS: @@ -253,3 +282,52 @@ def _remove_vault(self, address): self._experiments.pop(address, None) self.tags.pop(address, None) logger.debug('removed %s', address) + + +class RegistryEvents(ProcessedEvents[_EventItem]): + __slots__ = "_init_block", "_registry" + def __init__(self, registry: Registry, registries: List[Contract]): + self._init_block = chain.height + self._registry = registry + super().__init__(addresses=registries) + def _process_event(self, event: _EventItem) -> _EventItem: + # hack to make camels to snakes + event._ordered = [OrderedDict({inflection.underscore(k): v for k, v in od.items()}) for od in event._ordered] + logger.debug("starting to process %s for %s: %s", event.name, event.address, dict(event)) + if event.name == "NewGovernance": + self._registry.governance = event["governance"] + + if event.name == "NewRelease": + self._registry.releases[event["api_version"]] = contract(event["template"]) + + if event.name == "NewVault": + # experiment was endorsed + if event["vault"] in self._registry._experiments: + vault = self._registry._experiments.pop(event["vault"]) + vault.name = f"{vault.vault.symbol()} {event['api_version']}" + self._registry._vaults[event["vault"]] = vault + logger.debug("endorsed vault %s %s", vault.vault, vault.name) + # we already know this vault from another registry + elif event["vault"] not in self._registry._vaults: + vault = self._registry.vault_from_event(event) + vault.name = f"{vault.vault.symbol()} {event['api_version']}" + self._registry._vaults[event["vault"]] = vault + logger.debug("new vault %s %s", vault.vault, vault.name) + + if self._registry.include_experimental and event.name == "NewExperimentalVault": + vault = self._registry.vault_from_event(event) + vault.name = f"{vault.vault.symbol()} {event['api_version']} {event['vault'][:8]}" + self._registry._experiments[event["vault"]] = vault + logger.debug("new experiment %s %s", vault.vault, vault.name) + + if event.name == "VaultTagged": + if event["tag"] == "Removed": + self._registry._remove_vault(event["vault"]) + logger.debug("Removed vault %s", event["vault"]) + else: + self._registry.tags[event["vault"]] = event["tag"] + + if event.name == "StakingPoolAdded": + self._registry._staking_pools[event["token"]] = event["staking_pool"] + logger.debug("done processing %s for %s: %s", event.name, event.address, dict(event)) + return event diff --git a/yearn/v2/strategies.py b/yearn/v2/strategies.py index cf39f6371..445a6cb0b 100644 --- a/yearn/v2/strategies.py +++ b/yearn/v2/strategies.py @@ -1,16 +1,15 @@ import logging -import threading -import time from functools import cached_property -from typing import Any, List +from typing import Any, AsyncIterator, List -from brownie import chain +from async_property import async_property +from brownie.network.event import _EventItem from eth_utils import encode_hex, event_abi_to_log_topic from multicall.utils import run_in_subprocess -from y.exceptions import NodeNotSynced +from y import Contract +from y.decorators import stuck_coro_debugger +from y.utils.events import ProcessedEvents -from yearn.decorators import sentry_catch_all, wait_or_exit_after -from yearn.events import decode_logs, get_logs_asap from yearn.multicall2 import fetch_multicall_async from yearn.utils import contract, safe_views @@ -30,23 +29,9 @@ logger = logging.getLogger(__name__) -def _unpack_results(views: List[str], results: List[Any], scale: int): - # unpack self.vault.vault.strategies(self.strategy) - info = dict(zip(views, results)) - info.update(results[-1].dict()) - # scale views - for view in STRATEGY_VIEWS_SCALED: - if view in info: - info[view] = (info[view] or 0) / scale - # unwrap structs - for view in info: - if hasattr(info[view], '_dict'): - info[view] = info[view].dict() - return info - class Strategy: - def __init__(self, strategy, vault, watch_events_forever): + def __init__(self, strategy, vault): self.strategy = contract(strategy) self.vault = vault try: @@ -54,22 +39,11 @@ def __init__(self, strategy, vault, watch_events_forever): except ValueError: self.name = strategy[:10] self._views = safe_views(self.strategy.abi) - self._harvests = [] - self._topics = [ - [ - encode_hex(event_abi_to_log_topic(event)) - for event in self.strategy.abi - if event["type"] == "event" and event["name"] in STRATEGY_EVENTS - ] - ] - self._watch_events_forever = watch_events_forever - self._done = threading.Event() - self._has_exception = False - self._thread = threading.Thread(target=self.watch_events, daemon=True) + self._events = Harvests(self.strategy) - @property - def unique_name(self): - if [strategy.name for strategy in self.vault.strategies].count(self.name) > 1: + @async_property + async def unique_name(self): + if [strategy.name for strategy in await self.vault.strategies].count(self.name) > 1: return f'{self.name} {str(self.strategy)[:8]}' else: return self.name @@ -80,60 +54,52 @@ def __repr__(self) -> str: def __eq__(self, other): if isinstance(other, Strategy): return self.strategy == other.strategy - if isinstance(other, str): return self.strategy == other - raise ValueError("Strategy is only comparable with [Strategy, str]") - @sentry_catch_all - def watch_events(self): - start = time.time() - sleep_time = 300 - from_block = None - height = chain.height - while True: - logs = get_logs_asap(str(self.strategy), topics=self._topics, from_block=from_block, to_block=height) - events = decode_logs(logs) - self.process_events(events) - if not self._done.is_set(): - self._done.set() - logger.info("loaded %d harvests %s in %.3fs", len(self._harvests), self.name, time.time() - start) - if not self._watch_events_forever: - return - time.sleep(sleep_time) - - # read new logs at end of loop - from_block = height + 1 - height = chain.height - if height < from_block: - raise NodeNotSynced(f"No new blocks in the past {sleep_time/60} minutes.") - - - def process_events(self, events): - for event in events: - if event.name == "Harvested": - block = event.block_number - logger.debug("%s harvested on %d", self.name, block) - self._harvests.append(block) - - @wait_or_exit_after - def load_harvests(self): - if not self._thread._started.is_set(): - self._thread.start() - - @property - def harvests(self) -> List[int]: - self.load_harvests() - return self._harvests + async def harvests(self, thru_block: int) -> AsyncIterator[dict]: + async for event in self._events.events(to_block=thru_block): + yield event + @stuck_coro_debugger + async def describe(self, block=None): + results = await fetch_multicall_async(*self._calls, block=block) + return await self._unpack_results(results) + @cached_property def _calls(self): return *[[self.strategy, view] for view in self._views], [self.vault.vault, "strategies", self.strategy], async def _unpack_results(self, results): return await run_in_subprocess(_unpack_results, self._views, results, self.vault.scale) + + +class Harvests(ProcessedEvents[int]): + def __init__(self, strategy: Contract): + topics = [ + [ + encode_hex(event_abi_to_log_topic(event)) + for event in strategy.abi + if event["type"] == "event" and event["name"] in STRATEGY_EVENTS + ] + ] + super().__init__(addresses=[str(strategy)], topics=topics) + def _include_event(self, event: _EventItem) -> bool: + return event.name == "Harvested" + # TODO: work this in somehow: + # logger.info("loaded %d harvests %s in %.3fs", len(self._harvests), self.name, time.time() - start) + def _process_event(self, event: _EventItem) -> int: + block = event.block_number + logger.debug("%s harvested on %d", self.name, block) + return - async def describe(self, block=None): - results = await fetch_multicall_async(*self._calls, block=block) - return await self._unpack_results(results) + +def _unpack_results(views: List[str], results: List[Any], scale: int): + # unpack self.vault.vault.strategies(self.strategy) + info = dict(zip(views, results)) + info.update(results[-1].dict()) + # scale views + info = {view: (result or 0) / scale if view in STRATEGY_VIEWS_SCALED else result for view, result in info.items()} + # unwrap structs + return {view: result.dict() if hasattr(info[view], '_dict') else result for view, result in info.items()} \ No newline at end of file diff --git a/yearn/yearn.py b/yearn/yearn.py index 7f0ec550c..9319ad529 100644 --- a/yearn/yearn.py +++ b/yearn/yearn.py @@ -6,6 +6,7 @@ from brownie import chain from y.contracts import contract_creation_block_async +from y.decorators import stuck_coro_debugger from y.networks import Network import yearn.iearn @@ -28,37 +29,32 @@ class Yearn: Can describe all products. """ - def __init__(self, load_strategies=True, load_harvests=False, load_transfers=False, watch_events_forever=True, exclude_ib_tvl=True) -> None: + def __init__(self, exclude_ib_tvl=True) -> None: start = time() if chain.id == Network.Mainnet: self.registries = { "earn": yearn.iearn.Registry(), "v1": yearn.v1.registry.Registry(), - "v2": yearn.v2.registry.Registry(watch_events_forever=watch_events_forever), + "v2": yearn.v2.registry.Registry(), "ib": yearn.ironbank.Registry(exclude_ib_tvl=exclude_ib_tvl), "special": yearn.special.Registry(), } elif chain.id in [Network.Gnosis, Network.Base]: self.registries = { - "v2": yearn.v2.registry.Registry(watch_events_forever=watch_events_forever), + "v2": yearn.v2.registry.Registry(), } elif chain.id in [Network.Fantom, Network.Arbitrum, Network.Optimism]: self.registries = { - "v2": yearn.v2.registry.Registry(watch_events_forever=watch_events_forever), + "v2": yearn.v2.registry.Registry(), "ib": yearn.ironbank.Registry(exclude_ib_tvl=exclude_ib_tvl), } else: raise UnsupportedNetwork('yearn is not supported on this network') self.exclude_ib_tvl = exclude_ib_tvl - - if load_strategies: - self.registries["v2"].load_strategies() - if load_harvests: - self.registries["v2"].load_harvests() logger.info('loaded yearn in %.3fs', time() - start) - + @stuck_coro_debugger async def active_vaults_at(self, block=None): active_vaults_by_registry = await asyncio.gather(*[registry.active_vaults_at(block) for registry in self.registries.values()]) active = [vault for registry in active_vaults_by_registry for vault in registry] @@ -71,14 +67,14 @@ async def active_vaults_at(self, block=None): return active - + @stuck_coro_debugger async def describe(self, block=None): if block is None: block = chain.height desc = await asyncio.gather(*[self.registries[key].describe(block=block) for key in self.registries]) return dict(zip(self.registries, desc)) - + @stuck_coro_debugger async def describe_wallets(self, block=None): from yearn.outputs.describers.registry import RegistryWalletDescriber describer = RegistryWalletDescriber() @@ -102,12 +98,12 @@ async def describe_wallets(self, block=None): data.update(agg_stats) return data - + @stuck_coro_debugger async def total_value_at(self, block=None): desc = await asyncio.gather(*[self.registries[key].total_value_at(block=block) for key in self.registries]) return dict(zip(self.registries, desc)) - + @stuck_coro_debugger async def data_for_export(self, block, timestamp) -> List: start = time() data = await self.describe(block) @@ -183,7 +179,7 @@ async def data_for_export(self, block, timestamp) -> List: return metrics_to_export - + @stuck_coro_debugger async def wallet_data_for_export(self, block: int, timestamp: int): data = await self.describe_wallets(block) metrics_to_export = [] From aa5296dc1c54df279503fe3086edb5952b7c5d99 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 23/59] feat: more stuff --- yearn/partners/snapshot.py | 7 +- yearn/prices/curve.py | 15 +- yearn/treasury/accountant/__init__.py | 4 + yearn/treasury/accountant/revenue/fees.py | 10 +- yearn/v2/vaults.py | 278 ++++++++++++---------- 5 files changed, 179 insertions(+), 135 deletions(-) diff --git a/yearn/partners/snapshot.py b/yearn/partners/snapshot.py index a4910d4b8..ff15adf4b 100644 --- a/yearn/partners/snapshot.py +++ b/yearn/partners/snapshot.py @@ -232,12 +232,13 @@ class WildcardWrapper: async def unwrap(self) -> List[Wrapper]: registry = Registry() wrappers = [self.wrapper] if isinstance(self.wrapper, str) else self.wrapper + vaults = await registry.vaults topics = construct_event_topic_set( - filter_by_name('Transfer', registry.vaults[0].vault.abi)[0], + filter_by_name('Transfer', vaults[0].vault.abi)[0], web3.codec, {'receiver': wrappers}, ) - addresses = [str(vault.vault) for vault in registry.vaults] + addresses = [str(vault.vault) for vault in vaults] from_block = min(await asyncio.gather(*[threads.run(contract_creation_block, address) for address in addresses])) # wrapper -> {vaults} @@ -249,7 +250,7 @@ async def unwrap(self) -> List[Wrapper]: return [ Wrapper(name=vault.name, vault=str(vault.vault), wrapper=wrapper) for wrapper in wrappers - for vault in registry.vaults + for vault in vaults if str(vault.vault) in deposits[wrapper] ] diff --git a/yearn/prices/curve.py b/yearn/prices/curve.py index f93649ebb..be48c06d4 100644 --- a/yearn/prices/curve.py +++ b/yearn/prices/curve.py @@ -23,9 +23,8 @@ from brownie import ZERO_ADDRESS, Contract, chain, convert, interface from brownie.convert import to_address from brownie.convert.datatypes import EthAddress -from cachetools.func import lru_cache, ttl_cache -from y.constants import EEE_ADDRESS -from y.exceptions import NodeNotSynced, PriceError +from cachetools.func import lru_cache +from y.exceptions import NodeNotSynced from y.networks import Network from y.prices import magic @@ -35,7 +34,7 @@ from yearn.exceptions import UnsupportedNetwork from yearn.multicall2 import fetch_multicall, fetch_multicall_async from yearn.typing import Address, AddressOrContract, Block -from yearn.utils import Singleton, contract, get_event_loop +from yearn.utils import Singleton, contract logger = logging.getLogger(__name__) @@ -101,6 +100,7 @@ class Ids(IntEnum): class CurveRegistry(metaclass=Singleton): # NOTE: before deprecating, figure out why this loads more pools than ypm + @wait_or_exit_after def __init__(self) -> None: if chain.id not in curve_contracts: raise UnsupportedNetwork("curve is not supported on this network") @@ -119,11 +119,12 @@ def __init__(self) -> None: self._done = threading.Event() self._thread = threading.Thread(target=self.watch_events, daemon=True) + self._thread.start() self._has_exception = False - @wait_or_exit_after def ensure_loaded(self): if not self._thread._started.is_set(): + logger.debug("starting thread") self._thread.start() @sentry_catch_all @@ -399,7 +400,7 @@ async def calculate_apy(self, gauge: Contract, lp_token: AddressOrContract, bloc try: rate = ( inflation_rate * relative_weight * 86400 * 365 / working_supply * 0.4 - ) / token_price + ) / float(token_price) except ZeroDivisionError: rate = 0 @@ -409,7 +410,7 @@ async def calculate_apy(self, gauge: Contract, lp_token: AddressOrContract, bloc "inflation rate": inflation_rate, "virtual price": virtual_price, "crv reward rate": rate, - "crv apy": rate * crv_price, + "crv apy": rate * float(crv_price), "token price": token_price, } diff --git a/yearn/treasury/accountant/__init__.py b/yearn/treasury/accountant/__init__.py index 9cb16c3d2..4544ec065 100644 --- a/yearn/treasury/accountant/__init__.py +++ b/yearn/treasury/accountant/__init__.py @@ -1,4 +1,6 @@ +import logging + from brownie import Contract from brownie.exceptions import ContractNotFound from tqdm import tqdm @@ -9,6 +11,8 @@ from yearn.treasury.accountant.prepare_db import prepare_db from yearn.utils import contract +logger = logging.getLogger(__name__) + __all__ = [ "all_txs", "unsorted_txs", diff --git a/yearn/treasury/accountant/revenue/fees.py b/yearn/treasury/accountant/revenue/fees.py index 574274f0c..e197f676b 100644 --- a/yearn/treasury/accountant/revenue/fees.py +++ b/yearn/treasury/accountant/revenue/fees.py @@ -1,4 +1,6 @@ +import asyncio +import logging from brownie import chain from y.networks import Network @@ -41,13 +43,19 @@ def is_fees_v1(tx: TreasuryTx) -> bool: return False +_vaults = asyncio.get_event_loop().run_until_complete(v2.vaults) +_experiments = asyncio.get_event_loop().run_until_complete(v2.experiments) +v2_vaults = _vaults + _experiments +logger = logging.getLogger(__name__) +logger.info('%s v2 vaults loaded', len(v2_vaults)) + def is_fees_v2(tx: TreasuryTx) -> bool: if any( tx.from_address.address == vault.vault.address and tx.token.address.address == vault.vault.address and tx.to_address.address in treasury.addresses and tx.to_address.address == vault.vault.rewards(block_identifier=tx.block) - for vault in v2.vaults + v2.experiments + for vault in v2_vaults ): return True elif is_inverse_fees_from_stash_contract(tx): diff --git a/yearn/v2/vaults.py b/yearn/v2/vaults.py index 4242f6c63..29e3629b4 100644 --- a/yearn/v2/vaults.py +++ b/yearn/v2/vaults.py @@ -1,29 +1,33 @@ import asyncio import logging import re -import threading import time -from typing import TYPE_CHECKING, Any, Dict, List, Union +from functools import cached_property +from typing import (TYPE_CHECKING, Any, AsyncIterator, Dict, List, NoReturn, + Optional, Union) -from async_property import async_cached_property +import a_sync +from async_property import async_cached_property, async_property from brownie import chain +from brownie.network.event import _EventItem from eth_utils import encode_hex, event_abi_to_log_topic -from joblib import Parallel, delayed from multicall.utils import run_in_subprocess from semantic_version.base import Version from y import ERC20, Contract, Network, magic -from y.exceptions import NodeNotSynced, PriceError, yPriceMagicError +from y.contracts import contract_creation_block_async +from y.decorators import stuck_coro_debugger +from y.exceptions import PriceError, yPriceMagicError from y.networks import Network from y.prices import magic -from y.utils.events import get_logs_asap +from y.utils.dank_mids import dank_w3 +from y.utils.events import ProcessedEvents from yearn.common import Tvl -from yearn.decorators import sentry_catch_all, wait_or_exit_after -from yearn.events import decode_logs, get_logs_asap +from yearn.decorators import set_exc from yearn.multicall2 import fetch_multicall_async from yearn.special import Ygov from yearn.typing import Address -from yearn.utils import run_in_thread, safe_views +from yearn.utils import safe_views from yearn.v2.strategies import Strategy if TYPE_CHECKING: @@ -91,7 +95,7 @@ def _unpack_results(vault: Address, is_experiment: bool, _views: List[str], resu info["token price"] = float(price) if "totalAssets" in info: - info["tvl"] = info["token price"] * info["totalAssets"] + info["tvl"] = float(info["token price"]) * info["totalAssets"] for strategy_name, desc in zip(strategies, strategy_descs): info["strategies"][strategy_name] = desc @@ -101,9 +105,9 @@ def _unpack_results(vault: Address, is_experiment: bool, _views: List[str], resu info["version"] = "v2" return info - + class Vault: - def __init__(self, vault: Contract, api_version=None, token=None, registry=None, watch_events_forever=True): + def __init__(self, vault: Contract, api_version=None, token=None, registry=None): self._strategies: Dict[Address, Strategy] = {} self._revoked: Dict[Address, Strategy] = {} self._reports = [] @@ -116,24 +120,17 @@ def __init__(self, vault: Contract, api_version=None, token=None, registry=None, self.scale = 10 ** self.vault.decimals() # multicall-safe views with 0 inputs and numeric output. self._views = safe_views(self.vault.abi) + self._calls = [[self.vault, view] for view in self._views] # load strategies from events and watch for freshly attached strategies - self._topics = [ - [ - encode_hex(event_abi_to_log_topic(event)) - for event in self.vault.abi - if event["type"] == "event" and event["name"] in STRATEGY_EVENTS - ] - ] - self._watch_events_forever = watch_events_forever - self._done = threading.Event() - self._has_exception = False - self._thread = threading.Thread(target=self.watch_events, daemon=True) + self._events = VaultEvents(self) + self._done = a_sync.Event(name=f"{self.__module__}.{self.__class__.__name__}._done") + self._task = None def __repr__(self): strategies = "..." # don't block if we don't have the strategies loaded if self._done.is_set(): - strategies = ", ".join(f"{strategy}" for strategy in self.strategies) + strategies = ", ".join(f"{strategy}" for strategy in self._strategies) return f'' def __eq__(self, other): @@ -156,124 +153,92 @@ def from_address(cls, address): instance.name = vault.name() return instance - @property - def strategies(self) -> List[Strategy]: - self.load_strategies() + @async_property + @stuck_coro_debugger + async def strategies(self) -> List[Strategy]: + await self.load_strategies() return list(self._strategies.values()) - - @property - def revoked_strategies(self) -> List[Strategy]: - self.load_strategies() + + async def strategies_at_block(self, block: int) -> AsyncIterator[Strategy]: + self._task + working = {} + async for _ in self._events.events(to_block=block): + for address in self._strategies: + if address in working: + continue + working[address] = asyncio.create_task(contract_creation_block_async(address, when_no_history_return_0=True)) + for address in list(working.keys()): + if working[address].done(): + if await working.pop(address) > block: + return + yield self._strategies[address] + + await self._events._lock.wait_for(block) + + @async_property + @stuck_coro_debugger + async def revoked_strategies(self) -> List[Strategy]: + await self.load_strategies() return list(self._revoked.values()) - @property - def reports(self): + @async_property + @stuck_coro_debugger + async def reports(self): # strategy reports are loaded at the same time as other vault strategy events - self.load_strategies() + await self.load_strategies() return self._reports - @property - def is_endorsed(self): + @async_property + @stuck_coro_debugger + async def is_endorsed(self): if not self.registry: return None - return str(self.vault) in self.registry.vaults + return str(self.vault) in await self.registry.vaults - @property - def is_experiment(self): + @async_property + @stuck_coro_debugger + async def is_experiment(self): if not self.registry: return None # experimental vaults are either listed in the registry or have the 0x address suffix in the name - return str(self.vault) in self.registry.experiments or re.search(r"0x.*$", self.name) is not None + return str(self.vault) in await self.registry.experiments or re.search(r"0x.*$", self.name) is not None - @wait_or_exit_after - def load_strategies(self): - if not self._thread._started.is_set(): - self._thread.start() - - def load_harvests(self): - Parallel(1, "threading")(delayed(strategy.load_harvests)() for strategy in self.strategies) - - @sentry_catch_all - def watch_events(self): + @stuck_coro_debugger + async def is_active(self, block: Optional[int]) -> bool: + if block and await contract_creation_block_async(str(self.vault)) < block: + return False + # fixes edge case: a vault is not necessarily initialized on creation + return self.vault.activation.coroutine(block_identifier=block) + + @stuck_coro_debugger + async def load_strategies(self): + self._task + await self._done.wait() + if self._task.done() and (e := self._task.exception()): + raise e + + @set_exc + async def watch_events(self) -> NoReturn: start = time.time() - sleep_time = 300 - from_block = None - height = chain.height - while True: - logs = get_logs_asap(str(self.vault), topics=self._topics, from_block=from_block, to_block=height) - events = decode_logs(logs) - self.process_events(events) - if not self._done.is_set(): - self._done.set() - logger.info("loaded %d strategies %s in %.3fs", len(self._strategies), self.name, time.time() - start) - if not self._watch_events_forever: - return - time.sleep(sleep_time) - - # set vars for next loop - from_block = height + 1 - height = chain.height - if height < from_block: - raise NodeNotSynced(f"No new blocks in the past {sleep_time/60} minutes.") - - - def process_events(self, events): - for event in events: - # some issues during the migration of this strat prevented it from being verified so we skip it here... - if chain.id == Network.Optimism: - failed_migration = False - for key in ["newVersion", "oldVersion", "strategy"]: - failed_migration |= (key in event and event[key] == "0x4286a40EB3092b0149ec729dc32AD01942E13C63") - if failed_migration: - continue - - if event.name == "StrategyAdded": - strategy_address = event["strategy"] - logger.debug("%s strategy added %s", self.name, strategy_address) - try: - self._strategies[strategy_address] = Strategy(strategy_address, self, self._watch_events_forever) - except ValueError: - logger.error(f"Error loading strategy {strategy_address}") - pass - elif event.name == "StrategyRevoked": - logger.debug("%s strategy revoked %s", self.name, event["strategy"]) - self._revoked[event["strategy"]] = self._strategies.pop( - event["strategy"], Strategy(event["strategy"], self, self._watch_events_forever) - ) - elif event.name == "StrategyMigrated": - logger.debug("%s strategy migrated %s -> %s", self.name, event["oldVersion"], event["newVersion"]) - self._revoked[event["oldVersion"]] = self._strategies.pop( - event["oldVersion"], Strategy(event["oldVersion"], self, self._watch_events_forever) - ) - self._strategies[event["newVersion"]] = Strategy(event["newVersion"], self, self._watch_events_forever) - elif event.name == "StrategyReported": - self._reports.append(event) - - async def _unpack_results(self, results): - results, strategy_descs, price = results - strategies = await run_in_thread(getattr, self, 'strategies') - return await run_in_subprocess( - _unpack_results, - self.vault.address, - self.is_experiment, - self._views, - results, - self.scale, - price, - # must be picklable. - [strategy.unique_name for strategy in strategies], - strategy_descs, - ) - + height = await dank_w3.eth.block_number + done_task = asyncio.create_task(self._events._lock.wait_for(height)) + def done_callback(task: asyncio.Task) -> None: + logger.info("loaded %d strategies %s in %.3fs", len(self._strategies), self.name, time.time() - start) + self._done.set() + done_task.add_done_callback(done_callback) + self._events._ensure_task() + + @stuck_coro_debugger async def describe(self, block=None): - strategies = await run_in_thread(getattr, self, 'strategies') + block = block or await dank_w3.eth.block_number results = await asyncio.gather( - fetch_multicall_async(*[[self.vault, view] for view in self._views], block=block), - asyncio.gather(*[strategy.describe(block=block) for strategy in strategies]), - get_price_return_exceptions(self.token, block=block) + fetch_multicall_async(*self._calls, block=block), + self._describe_strategies(block), + get_price_return_exceptions(self.token, block=block), ) return await self._unpack_results(results) - + + @stuck_coro_debugger async def apy(self, samples: "ApySamples"): from yearn import apy if self._needs_curve_simple: @@ -287,6 +252,7 @@ async def apy(self, samples: "ApySamples"): else: return await apy.v2.simple(self, samples) + @stuck_coro_debugger async def tvl(self, block=None): total_assets = await self.vault.totalAssets.coroutine(block_identifier=block) try: @@ -303,7 +269,12 @@ async def tvl(self, block=None): tvl = total_assets * price / await ERC20(self.vault, asynchronous=True).scale if price else None return Tvl(total_assets, price, tvl) + @cached_property + def _task(self) -> asyncio.Task: + return asyncio.create_task(self.watch_events()) + @async_cached_property + @stuck_coro_debugger async def _needs_curve_simple(self): # some curve vaults which should not be calculated with curve logic curve_simple_excludes = { @@ -316,3 +287,62 @@ async def _needs_curve_simple(self): needs_simple = self.vault.address not in curve_simple_excludes[chain.id] return needs_simple and magic.curve and await magic.curve.get_pool(self.token.address) + + @stuck_coro_debugger + async def _describe_strategies(self, block: int) -> List[dict]: + return asyncio.gather(*[asyncio.create_task(strategy.describe(block=block)) async for strategy in self.strategies_at_block(block)]) + + @stuck_coro_debugger + async def _unpack_results(self, results): + results, strategy_descs, price = results + return await run_in_subprocess( + _unpack_results, + self.vault.address, + await self.is_experiment, + self._views, + results, + self.scale, + price, + # must be picklable. + [strategy.unique_name for strategy in await self.strategies], + strategy_descs, + ) + + +class VaultEvents(ProcessedEvents[int]): + __slots__ = "vault", + def __init__(self, vault: Vault, **kwargs: Any): + topics = [[encode_hex(event_abi_to_log_topic(event)) for event in vault.vault.abi if event["type"] == "event" and event["name"] in STRATEGY_EVENTS]] + super().__init__(addresses=[str(vault.vault)], topics=topics, **kwargs) + self.vault = vault + def _process_event(self, event: _EventItem) -> _EventItem: + # some issues during the migration of this strat prevented it from being verified so we skip it here... + if chain.id == Network.Optimism: + failed_migration = False + for key in ["newVersion", "oldVersion", "strategy"]: + failed_migration |= (key in event and event[key] == "0x4286a40EB3092b0149ec729dc32AD01942E13C63") + if failed_migration: + return event + + if event.name == "StrategyAdded": + strategy_address = event["strategy"] + logger.debug("%s strategy added %s", self.vault.name, strategy_address) + try: + self.vault._strategies[strategy_address] = Strategy(strategy_address, self.vault) + except ValueError: + logger.error(f"Error loading strategy {strategy_address}") + pass + elif event.name == "StrategyRevoked": + logger.debug("%s strategy revoked %s", self.vault.name, event["strategy"]) + self.vault._revoked[event["strategy"]] = self.vault._strategies.pop( + event["strategy"], Strategy(event["strategy"], self.vault) + ) + elif event.name == "StrategyMigrated": + logger.debug("%s strategy migrated %s -> %s", self.vault.name, event["oldVersion"], event["newVersion"]) + self.vault._revoked[event["oldVersion"]] = self.vault._strategies.pop( + event["oldVersion"], Strategy(event["oldVersion"], self.vault) + ) + self.vault._strategies[event["newVersion"]] = Strategy(event["newVersion"], self.vault) + elif event.name == "StrategyReported": + self.vault._reports.append(event) + return event \ No newline at end of file From 38a3a68ed12e955b524af4bd7f690c852982d527 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 24/59] chore: update .env.example --- .env.example | 1 + 1 file changed, 1 insertion(+) diff --git a/.env.example b/.env.example index d47dbd711..e123c35dc 100644 --- a/.env.example +++ b/.env.example @@ -35,6 +35,7 @@ export CONCURRENCY= # 1,2,3... Positive whole number for how many blocks get pro export YPRICEAPI_URL= # YPriceAPI url export YPRICEAPI_SIGNER= # Address you signed up for yPriceAPI on export YPRICEAPI_SIGNATURE= # Signature from subscription +export YPRICEMAGIC_GETLOGS_DOP= # Max number of concurrent eth_getLogs calls export YPRICEMAGIC_RECURSION_TIMEOUT= # Time in seconds till yPriceMagic Loop for pricing times out without returning data. export SKIP_YPRICEAPI= # False or True. Defaults to True. From 7c574a5d8e35d878a8ee7d70a326c426188af949 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 25/59] feat: more stuff --- scripts/exporters/transactions.py | 2 +- services/dashboard/docker-compose.infra.yml | 14 ++++++++++++ yearn/decorators.py | 8 +++---- yearn/treasury/accountant/ignore/__init__.py | 2 +- yearn/treasury/accountant/ignore/vaults.py | 6 ++--- yearn/v2/registry.py | 23 ++++++++++++++------ yearn/v2/vaults.py | 8 +++++-- 7 files changed, 45 insertions(+), 18 deletions(-) diff --git a/scripts/exporters/transactions.py b/scripts/exporters/transactions.py index 23752d034..f6b617a6e 100644 --- a/scripts/exporters/transactions.py +++ b/scripts/exporters/transactions.py @@ -29,7 +29,7 @@ warnings.simplefilter("ignore", BrownieEnvironmentWarning) -yearn = Yearn(load_strategies=False) +yearn = Yearn() logger = logging.getLogger('yearn.transactions_exporter') diff --git a/services/dashboard/docker-compose.infra.yml b/services/dashboard/docker-compose.infra.yml index 4fd50372d..f0ebc1fff 100644 --- a/services/dashboard/docker-compose.infra.yml +++ b/services/dashboard/docker-compose.infra.yml @@ -103,3 +103,17 @@ services: volumes: - postgres_data:/var/lib/postgresql/data restart: always + ypostgres: + image: postgres:14 + command: -c 'max_connections=${PGCONNECTIONS:-5000}' + ports: + - 5420:5432 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=yearn-exporter + - POSTGRES_DB=postgres + networks: + - stack + volumes: + - ypostgres_data:/var/lib/postgresql/data + restart: always diff --git a/yearn/decorators.py b/yearn/decorators.py index c1f1bd198..fb7020bb5 100644 --- a/yearn/decorators.py +++ b/yearn/decorators.py @@ -26,13 +26,13 @@ def wait_or_exit_before(func): @functools.wraps(func) async def wrap(self): task: asyncio.Task = self._task - logger.info("waiting for %s", self) + logger.debug("waiting for %s", self) while not self._done.is_set() and not task.done(): await asyncio.sleep(10) - logger.info("%s not done", self) - logger.info("loading %s complete", self) + logger.debug("%s not done", self) + logger.debug("loading %s complete", self) if task.done() and (e := task.exception()): - logger.info('task %s has exception %s, awaiting', task, e) + logger.debug('task %s has exception %s, awaiting', task, e) raise e return await func(self) return wrap diff --git a/yearn/treasury/accountant/ignore/__init__.py b/yearn/treasury/accountant/ignore/__init__.py index 1cf5f38cf..e78c83a5f 100644 --- a/yearn/treasury/accountant/ignore/__init__.py +++ b/yearn/treasury/accountant/ignore/__init__.py @@ -2,7 +2,7 @@ from decimal import Decimal from brownie import chain -from y.networks import Network +from y import Network from yearn.entities import TreasuryTx from yearn.treasury.accountant.classes import HashMatcher, TopLevelTxGroup diff --git a/yearn/treasury/accountant/ignore/vaults.py b/yearn/treasury/accountant/ignore/vaults.py index c682dd204..7fd9fedf8 100644 --- a/yearn/treasury/accountant/ignore/vaults.py +++ b/yearn/treasury/accountant/ignore/vaults.py @@ -1,13 +1,13 @@ - from brownie import ZERO_ADDRESS, chain from y.networks import Network from yearn.entities import TreasuryTx +from yearn.treasury.accountant.revenue.fees import v2_vaults from yearn.treasury.accountant.classes import Filter, HashMatcher, IterFilter -from yearn.treasury.accountant.constants import treasury, v1, v2 +from yearn.treasury.accountant.constants import treasury, v1 from yearn.utils import contract -vaults = (v1.vaults + v2.vaults) if v1 else v2.vaults +vaults = (v1.vaults + v2_vaults) if v1 else v2_vaults def is_vault_deposit(tx: TreasuryTx) -> bool: """ This code doesn't validate amounts but so far that's not been a problem. """ diff --git a/yearn/v2/registry.py b/yearn/v2/registry.py index 5e65251ff..c9f7a3856 100644 --- a/yearn/v2/registry.py +++ b/yearn/v2/registry.py @@ -251,14 +251,23 @@ async def _active_vaults_at(self, block=None) -> List[Vault]: return [vault for vault, active in zip(vaults, await asyncio.gather(*[vault.is_active(block) for vault in vaults])) if active] async def _active_vaults_at_iter(self, block=None) -> AsyncIterator[Vault]: + # ensure loader task is running self._task events = await self._events + # make sure the events are loaded thru now before proceeding await events._lock.wait_for(events._init_block) - vaults = list(itertools.chain(self._vaults.values(), self._experiments.values())) - async def is_active(vault: Vault) -> bool: - return vault, await vault.is_active(block) - for fut in asyncio.as_completed([is_active(vault) for vault in vaults]): - vault, active = await fut + + vaults: List[Vault] = list(itertools.chain(self._vaults.values(), self._experiments.values())) + + i = 0 # TODO figure out why we need this here + while len(vaults) == 0: + await asyncio.sleep(6) + vaults = list(itertools.chain(self._vaults.values(), self._experiments.values())) + i += 1 + if i >= 20: + logger.error("we're stuck") + + async for vault, active in a_sync.as_completed({vault: vault.is_active(block) for vault in vaults}, aiter=True): if active: yield vault @@ -271,11 +280,11 @@ def _task(self) -> asyncio.Task: return asyncio.create_task(self.watch_events()) def _filter_vaults(self): - logger.debug('filtering vaults') + #logger.debug('filtering vaults') if chain.id in DEPRECATED_VAULTS: for vault in DEPRECATED_VAULTS[chain.id]: self._remove_vault(vault) - logger.debug('vaults filtered') + #logger.debug('vaults filtered') def _remove_vault(self, address): self._vaults.pop(address, None) diff --git a/yearn/v2/vaults.py b/yearn/v2/vaults.py index 29e3629b4..04b909e44 100644 --- a/yearn/v2/vaults.py +++ b/yearn/v2/vaults.py @@ -133,6 +133,9 @@ def __repr__(self): strategies = ", ".join(f"{strategy}" for strategy in self._strategies) return f'' + def __hash__(self) -> int: + return hash(self.vault.address) + def __eq__(self, other): if isinstance(other, Vault): return self.vault == other.vault @@ -205,10 +208,10 @@ async def is_experiment(self): @stuck_coro_debugger async def is_active(self, block: Optional[int]) -> bool: - if block and await contract_creation_block_async(str(self.vault)) < block: + if block and await contract_creation_block_async(str(self.vault)) > block: return False # fixes edge case: a vault is not necessarily initialized on creation - return self.vault.activation.coroutine(block_identifier=block) + return await self.vault.activation.coroutine(block_identifier=block) @stuck_coro_debugger async def load_strategies(self): @@ -294,6 +297,7 @@ async def _describe_strategies(self, block: int) -> List[dict]: @stuck_coro_debugger async def _unpack_results(self, results): + # TODO: get rid of this results, strategy_descs, price = results return await run_in_subprocess( _unpack_results, From 508a9f2a51a388896af9904c154edfaba3380181 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 26/59] feat: ydb envs --- services/dashboard/docker-compose.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/dashboard/docker-compose.yml b/services/dashboard/docker-compose.yml index d29ce7771..4fb0cadf9 100644 --- a/services/dashboard/docker-compose.yml +++ b/services/dashboard/docker-compose.yml @@ -110,6 +110,13 @@ x-envs: &envs # DOCKER CONTAINER ENVS - CONTAINER_NAME + + - YPRICEMAGIC_DB_PROVIDER=postgres + - YPRICEMAGIC_DB_HOST=ypostgres + - YPRICEMAGIC_DB_PORT=5432 + - YPRICEMAGIC_DB_USER=${PGUSER:-postgres} + - YPRICEMAGIC_DB_PASSWORD=${PGPASSWORD:-yearn-exporter} + - YPRICEMAGIC_DB_DATABASE=${YPRICEMAGIC_DB_DATABASE:-postgres} x-volumes: &volumes - brownie:/root/.brownie From 17eb60f5c7465978b73cf2566ed7bc1b63c15808 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 27/59] feat: ydb link --- services/dashboard/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/services/dashboard/docker-compose.yml b/services/dashboard/docker-compose.yml index 4fb0cadf9..c8f0cc5f5 100644 --- a/services/dashboard/docker-compose.yml +++ b/services/dashboard/docker-compose.yml @@ -133,6 +133,7 @@ services: - yearn-exporter-infra_stack external_links: - yearn-exporter-infra-postgres-1:postgres + - yearn-exporter-infra_ypostgres_1:ypostgres - yearn-exporter-infra-victoria-metrics-1:victoria-metrics logging: driver: "json-file" From 0db72b728916b3f2604c6758653283a0e7c0685f Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 28/59] fix: missing volume --- services/dashboard/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/services/dashboard/docker-compose.yml b/services/dashboard/docker-compose.yml index c8f0cc5f5..fb34d5556 100644 --- a/services/dashboard/docker-compose.yml +++ b/services/dashboard/docker-compose.yml @@ -4,6 +4,7 @@ volumes: brownie: {} cache: {} memray: {} + ypostgres: {} ypricemagic: {} networks: From 7066e871812189f3c240750c33503b81e4841477 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 29/59] fix: missing volume --- services/dashboard/docker-compose.infra.yml | 1 + services/dashboard/docker-compose.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dashboard/docker-compose.infra.yml b/services/dashboard/docker-compose.infra.yml index f0ebc1fff..5bde03cea 100644 --- a/services/dashboard/docker-compose.infra.yml +++ b/services/dashboard/docker-compose.infra.yml @@ -4,6 +4,7 @@ volumes: grafana_data: {} victoria_metrics_data: {} postgres_data: {} + ypostgres_data: {} networks: stack: diff --git a/services/dashboard/docker-compose.yml b/services/dashboard/docker-compose.yml index fb34d5556..c8f0cc5f5 100644 --- a/services/dashboard/docker-compose.yml +++ b/services/dashboard/docker-compose.yml @@ -4,7 +4,6 @@ volumes: brownie: {} cache: {} memray: {} - ypostgres: {} ypricemagic: {} networks: From ea6a617ccc3a2e7d53e880b7676b80c159d6f303 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 30/59] fix: change underscore to hyphen --- services/dashboard/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dashboard/docker-compose.yml b/services/dashboard/docker-compose.yml index c8f0cc5f5..17cf84c4b 100644 --- a/services/dashboard/docker-compose.yml +++ b/services/dashboard/docker-compose.yml @@ -133,7 +133,7 @@ services: - yearn-exporter-infra_stack external_links: - yearn-exporter-infra-postgres-1:postgres - - yearn-exporter-infra_ypostgres_1:ypostgres + - yearn-exporter-infra-ypostgres-1:ypostgres - yearn-exporter-infra-victoria-metrics-1:victoria-metrics logging: driver: "json-file" From 6db06678bb06483fe0fde9ae9fdcf22dbc1e2ba4 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 31/59] fix: missing await --- scripts/drome_apy_previews.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/drome_apy_previews.py b/scripts/drome_apy_previews.py index b89f56f52..0d1a4ead0 100644 --- a/scripts/drome_apy_previews.py +++ b/scripts/drome_apy_previews.py @@ -85,7 +85,7 @@ async def _build_data(): async def _get_lps_with_vault_potential() -> List[dict]: sugar_oracle = await Contract.coroutine(drome.sugar) - current_vaults = Registry(include_experimental=False).vaults + current_vaults = await Registry(include_experimental=False).vaults current_underlyings = [str(vault.token) for vault in current_vaults] return [lp for lp in await sugar_oracle.all.coroutine(999999999999999999999, 0, ZERO_ADDRESS) if lp[0] not in current_underlyings and lp[11] != ZERO_ADDRESS] From 86ce1440c55ad57dbfdb75f5dbd99db911e9bd4b Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 32/59] fix: missing await --- yearn/v2/vaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yearn/v2/vaults.py b/yearn/v2/vaults.py index 04b909e44..552a52a8c 100644 --- a/yearn/v2/vaults.py +++ b/yearn/v2/vaults.py @@ -244,7 +244,7 @@ async def describe(self, block=None): @stuck_coro_debugger async def apy(self, samples: "ApySamples"): from yearn import apy - if self._needs_curve_simple: + if await self._needs_curve_simple: return await apy.curve.simple(self, samples) elif pool := await apy.velo.get_staking_pool(self.token.address): return await apy.velo.staking(self, pool, samples) From a2964fa135022c55bf427060924efa463e75bab9 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 33/59] fix: Decimal type err --- yearn/outputs/describers/vault.py | 9 +++++---- yearn/outputs/postgres/utils.py | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/yearn/outputs/describers/vault.py b/yearn/outputs/describers/vault.py index 962ffc37b..eeea7a061 100644 --- a/yearn/outputs/describers/vault.py +++ b/yearn/outputs/describers/vault.py @@ -1,6 +1,7 @@ import asyncio from concurrent.futures import ProcessPoolExecutor +from decimal import Decimal from yearn.outputs.postgres.utils import fetch_balances from yearn.prices.magic import _get_price @@ -26,10 +27,10 @@ async def describe_wallets(self, vault_address, block=None): 'total wallets': len(set(wallet for wallet, bal in balances.items())), 'wallet balances': { wallet: { - "token balance": float(bal), - "usd balance": float(bal) * price - } for wallet, bal in balances.items() - } + "token balance": bal, + "usd balance": bal * Decimal(price) + } for wallet, bal in balances.items() } + } info['active wallets'] = sum(1 if balances['usd balance'] > ACTIVE_WALLET_USD_THRESHOLD else 0 for balances in info['wallet balances'].values()) return info diff --git a/yearn/outputs/postgres/utils.py b/yearn/outputs/postgres/utils.py index eb083161f..8afc5de0a 100644 --- a/yearn/outputs/postgres/utils.py +++ b/yearn/outputs/postgres/utils.py @@ -1,5 +1,6 @@ import logging -from typing import Optional +from decimal import Decimal +from typing import Dict, Optional from brownie import ZERO_ADDRESS, chain, convert from brownie.convert.datatypes import HexString @@ -115,7 +116,7 @@ def last_recorded_block(Entity: db.Entity) -> int: return select(max(e.block) for e in Entity if e.chainid == chain.id).first() @db_session -def fetch_balances(vault_address: str, block=None): +def fetch_balances(vault_address: str, block=None) -> Dict[str, Decimal]: token_dbid = select(t.token_id for t in Token if t.chain.chainid == chain.id and t.address.address == vault_address).first() if block and block > last_recorded_block(UserTx): # NOTE: we use `postgres.` instead of `self.` so we can make use of parallelism From 8dd1cbf58e4889a930467125a87394820517ec24 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 34/59] fix: curve simple Decimal err --- yearn/apy/curve/simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yearn/apy/curve/simple.py b/yearn/apy/curve/simple.py index 5093a7b64..3ca75d90b 100644 --- a/yearn/apy/curve/simple.py +++ b/yearn/apy/curve/simple.py @@ -195,7 +195,7 @@ async def calculate_simple(vault, gauge: Gauge, samples: ApySamples) -> Apy: * gauge_weight * (SECONDS_PER_YEAR / gauge.gauge_working_supply) * (PER_MAX_BOOST / pool_price) - * crv_price + * float(crv_price) ) / base_asset_price if y_gauge_balance > 0: From 9154d471e259735bb606dc065e591bce3a2ea4a1 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 35/59] fix: load_strategies --- yearn/v2/registry.py | 2 - yearn/v2/vaults.py | 91 +++++++++++++++++++++----------------------- 2 files changed, 44 insertions(+), 49 deletions(-) diff --git a/yearn/v2/registry.py b/yearn/v2/registry.py index c9f7a3856..e8f0319e9 100644 --- a/yearn/v2/registry.py +++ b/yearn/v2/registry.py @@ -280,11 +280,9 @@ def _task(self) -> asyncio.Task: return asyncio.create_task(self.watch_events()) def _filter_vaults(self): - #logger.debug('filtering vaults') if chain.id in DEPRECATED_VAULTS: for vault in DEPRECATED_VAULTS[chain.id]: self._remove_vault(vault) - #logger.debug('vaults filtered') def _remove_vault(self, address): self._vaults.pop(address, None) diff --git a/yearn/v2/vaults.py b/yearn/v2/vaults.py index 552a52a8c..9ae0a19a8 100644 --- a/yearn/v2/vaults.py +++ b/yearn/v2/vaults.py @@ -2,11 +2,12 @@ import logging import re import time +from contextlib import suppress from functools import cached_property from typing import (TYPE_CHECKING, Any, AsyncIterator, Dict, List, NoReturn, Optional, Union) -import a_sync +from a_sync.utils.iterators import exhaust_iterator from async_property import async_cached_property, async_property from brownie import chain from brownie.network.event import _EventItem @@ -23,7 +24,6 @@ from y.utils.events import ProcessedEvents from yearn.common import Tvl -from yearn.decorators import set_exc from yearn.multicall2 import fetch_multicall_async from yearn.special import Ygov from yearn.typing import Address @@ -124,13 +124,12 @@ def __init__(self, vault: Contract, api_version=None, token=None, registry=None) # load strategies from events and watch for freshly attached strategies self._events = VaultEvents(self) - self._done = a_sync.Event(name=f"{self.__module__}.{self.__class__.__name__}._done") - self._task = None def __repr__(self): strategies = "..." # don't block if we don't have the strategies loaded - if self._done.is_set(): - strategies = ", ".join(f"{strategy}" for strategy in self._strategies) + with suppress(RuntimeError): # NOTE on RuntimeError, event loop isnt running and task doesn't exist. can't be created, is not complete. no need to check strats. + if self._task.done(): + strategies = ", ".join(f"{strategy}" for strategy in self._strategies) return f'' def __hash__(self) -> int: @@ -215,21 +214,16 @@ async def is_active(self, block: Optional[int]) -> bool: @stuck_coro_debugger async def load_strategies(self): - self._task - await self._done.wait() - if self._task.done() and (e := self._task.exception()): - raise e + await self._task - @set_exc async def watch_events(self) -> NoReturn: start = time.time() - height = await dank_w3.eth.block_number - done_task = asyncio.create_task(self._events._lock.wait_for(height)) - def done_callback(task: asyncio.Task) -> None: - logger.info("loaded %d strategies %s in %.3fs", len(self._strategies), self.name, time.time() - start) - self._done.set() - done_task.add_done_callback(done_callback) - self._events._ensure_task() + async for event in self._events.events(await dank_w3.eth.block_number): + # we iterate thru the list to ensure they're loaded thru now + pass + logger.info("loaded %d strategies %s in %.3fs", len(self._strategies), self.name, time.time() - start) + # Keep the loader running in the background + self._daemon = asyncio.create_task(exhaust_iterator(self._events)) @stuck_coro_debugger async def describe(self, block=None): @@ -313,7 +307,7 @@ async def _unpack_results(self, results): ) -class VaultEvents(ProcessedEvents[int]): +class VaultEvents(ProcessedEvents[_EventItem]): __slots__ = "vault", def __init__(self, vault: Vault, **kwargs: Any): topics = [[encode_hex(event_abi_to_log_topic(event)) for event in vault.vault.abi if event["type"] == "event" and event["name"] in STRATEGY_EVENTS]] @@ -321,32 +315,35 @@ def __init__(self, vault: Vault, **kwargs: Any): self.vault = vault def _process_event(self, event: _EventItem) -> _EventItem: # some issues during the migration of this strat prevented it from being verified so we skip it here... - if chain.id == Network.Optimism: - failed_migration = False - for key in ["newVersion", "oldVersion", "strategy"]: - failed_migration |= (key in event and event[key] == "0x4286a40EB3092b0149ec729dc32AD01942E13C63") - if failed_migration: - return event + try: + if chain.id == Network.Optimism: + failed_migration = False + for key in ["newVersion", "oldVersion", "strategy"]: + failed_migration |= (key in event and event[key] == "0x4286a40EB3092b0149ec729dc32AD01942E13C63") + if failed_migration: + return event - if event.name == "StrategyAdded": - strategy_address = event["strategy"] - logger.debug("%s strategy added %s", self.vault.name, strategy_address) - try: - self.vault._strategies[strategy_address] = Strategy(strategy_address, self.vault) - except ValueError: - logger.error(f"Error loading strategy {strategy_address}") - pass - elif event.name == "StrategyRevoked": - logger.debug("%s strategy revoked %s", self.vault.name, event["strategy"]) - self.vault._revoked[event["strategy"]] = self.vault._strategies.pop( - event["strategy"], Strategy(event["strategy"], self.vault) - ) - elif event.name == "StrategyMigrated": - logger.debug("%s strategy migrated %s -> %s", self.vault.name, event["oldVersion"], event["newVersion"]) - self.vault._revoked[event["oldVersion"]] = self.vault._strategies.pop( - event["oldVersion"], Strategy(event["oldVersion"], self.vault) - ) - self.vault._strategies[event["newVersion"]] = Strategy(event["newVersion"], self.vault) - elif event.name == "StrategyReported": - self.vault._reports.append(event) - return event \ No newline at end of file + if event.name == "StrategyAdded": + strategy_address = event["strategy"] + logger.debug("%s strategy added %s", self.vault.name, strategy_address) + try: + self.vault._strategies[strategy_address] = Strategy(strategy_address, self.vault) + except ValueError: + logger.error(f"Error loading strategy {strategy_address}") + elif event.name == "StrategyRevoked": + logger.debug("%s strategy revoked %s", self.vault.name, event["strategy"]) + self.vault._revoked[event["strategy"]] = self.vault._strategies.pop( + event["strategy"], Strategy(event["strategy"], self.vault) + ) + elif event.name == "StrategyMigrated": + logger.debug("%s strategy migrated %s -> %s", self.vault.name, event["oldVersion"], event["newVersion"]) + self.vault._revoked[event["oldVersion"]] = self.vault._strategies.pop( + event["oldVersion"], Strategy(event["oldVersion"], self.vault) + ) + self.vault._strategies[event["newVersion"]] = Strategy(event["newVersion"], self.vault) + elif event.name == "StrategyReported": + self.vault._reports.append(event) + return event + except Exception as e: + logger.exception(e) + raise e \ No newline at end of file From 77e0a525a9fd2154d72060ad53d370c71ac81fad Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 36/59] fix: missing s3 await --- yearn/apy/staking_rewards.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yearn/apy/staking_rewards.py b/yearn/apy/staking_rewards.py index de51ee2c8..8cb20baa4 100644 --- a/yearn/apy/staking_rewards.py +++ b/yearn/apy/staking_rewards.py @@ -14,10 +14,11 @@ async def get_staking_rewards_apr(vault, samples: ApySamples): return 0 vault_address = str(vault.vault) - if vault_address not in vault.registry.staking_pools: + staking_pools = await vault.registry.staking_pools + if vault_address not in staking_pools: return 0 - staking_pool = await Contract.coroutine(vault.registry.staking_pools[vault_address]) + staking_pool = await Contract.coroutine(staking_pools[vault_address]) if await staking_pool.periodFinish.coroutine() < now: return 0 From 334e2fa001a8327564010136e9a9b107543b758d Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 37/59] fix: missing awawit --- yearn/apy/velo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yearn/apy/velo.py b/yearn/apy/velo.py index 89892ff32..4565fa082 100644 --- a/yearn/apy/velo.py +++ b/yearn/apy/velo.py @@ -37,7 +37,7 @@ async def staking(vault: "Vault", staking_rewards: Contract, samples: ApySamples rate = await staking_rewards.rewardRate.coroutine(block_identifier=block) if hasattr(staking_rewards, "rewardRate") else 0 performance = await vault.vault.performanceFee.coroutine(block_identifier=block) / 1e4 if hasattr(vault.vault, "performanceFee") else 0 management = await vault.vault.managementFee.coroutine(block_identifier=block) / 1e4 if hasattr(vault.vault, "managementFee") else 0 - strats = vault.strategies + strats = await vault.strategies keep = await strats[0].strategy.localKeepVELO.coroutine(block_identifier=block) / 1e4 if hasattr(strats[0].strategy, "localKeepVELO") else 0 rate = rate * (1 - keep) fees = ApyFees(performance=performance, management=management, keep_velo=keep) From 9916e53e4a5306b2f7376ded413117d64ca86c09 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 38/59] get_s3 type err --- scripts/s3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/s3.py b/scripts/s3.py index 1185d7130..5cc7640d1 100644 --- a/scripts/s3.py +++ b/scripts/s3.py @@ -279,10 +279,10 @@ def _export(data, file_name, s3_path): def _get_s3s(): s3s = [] - aws_buckets = os.environ.get("AWS_BUCKET").split(";") - aws_endpoint_urls = os.environ.get("AWS_ENDPOINT_URL").split(";") - aws_keys = os.environ.get("AWS_ACCESS_KEY").split(";") - aws_secrets = os.environ.get("AWS_ACCESS_SECRET").split(";") + aws_buckets = os.environ.get("AWS_BUCKET", "").split(";") + aws_endpoint_urls = os.environ.get("AWS_ENDPOINT_URL", "").split(";") + aws_keys = os.environ.get("AWS_ACCESS_KEY", "").split(";") + aws_secrets = os.environ.get("AWS_ACCESS_SECRET", "").split(";") for i in range(len(aws_buckets)): aws_bucket = aws_buckets[i] From 3c9063a530b3e61a9f49af9ddbb5dc4ac322d4c6 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:34 +0000 Subject: [PATCH 39/59] fix: Harvests --- yearn/v2/strategies.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/yearn/v2/strategies.py b/yearn/v2/strategies.py index 445a6cb0b..06869a740 100644 --- a/yearn/v2/strategies.py +++ b/yearn/v2/strategies.py @@ -76,7 +76,7 @@ async def _unpack_results(self, results): class Harvests(ProcessedEvents[int]): - def __init__(self, strategy: Contract): + def __init__(self, strategy: Strategy): topics = [ [ encode_hex(event_abi_to_log_topic(event)) @@ -84,14 +84,15 @@ def __init__(self, strategy: Contract): if event["type"] == "event" and event["name"] in STRATEGY_EVENTS ] ] - super().__init__(addresses=[str(strategy)], topics=topics) + super().__init__(addresses=[str(strategy.strategy)], topics=topics) + self.strategy = strategy def _include_event(self, event: _EventItem) -> bool: return event.name == "Harvested" # TODO: work this in somehow: # logger.info("loaded %d harvests %s in %.3fs", len(self._harvests), self.name, time.time() - start) def _process_event(self, event: _EventItem) -> int: block = event.block_number - logger.debug("%s harvested on %d", self.name, block) + logger.debug("%s harvested on %d", self.strategy.name, block) return From b0d17278fd73580b242dcf9319cb0cace77ba407 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 40/59] fix: missing commit --- yearn/v2/strategies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yearn/v2/strategies.py b/yearn/v2/strategies.py index 06869a740..609c0abb6 100644 --- a/yearn/v2/strategies.py +++ b/yearn/v2/strategies.py @@ -39,7 +39,7 @@ def __init__(self, strategy, vault): except ValueError: self.name = strategy[:10] self._views = safe_views(self.strategy.abi) - self._events = Harvests(self.strategy) + self._events = Harvests(self) @async_property async def unique_name(self): @@ -80,7 +80,7 @@ def __init__(self, strategy: Strategy): topics = [ [ encode_hex(event_abi_to_log_topic(event)) - for event in strategy.abi + for event in strategy.strategy.abi if event["type"] == "event" and event["name"] in STRATEGY_EVENTS ] ] From 4ffb366da5fe586139cc9ecd97cfeba8542a855b Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 41/59] fix: type err --- yearn/v2/strategies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yearn/v2/strategies.py b/yearn/v2/strategies.py index 609c0abb6..d4bb45956 100644 --- a/yearn/v2/strategies.py +++ b/yearn/v2/strategies.py @@ -93,7 +93,7 @@ def _include_event(self, event: _EventItem) -> bool: def _process_event(self, event: _EventItem) -> int: block = event.block_number logger.debug("%s harvested on %d", self.strategy.name, block) - return + return event def _unpack_results(views: List[str], results: List[Any], scale: int): From a096e1ca51b232621010a759c16bd4eae1532e39 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 42/59] fix: registry loader time --- yearn/v2/registry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yearn/v2/registry.py b/yearn/v2/registry.py index e8f0319e9..1aeb7f7ce 100644 --- a/yearn/v2/registry.py +++ b/yearn/v2/registry.py @@ -144,12 +144,12 @@ def __repr__(self) -> str: @set_exc async def watch_events(self) -> NoReturn: - start = await dank_w3.eth.block_number + start = time.time() events = await self._events def done_callback(task: asyncio.Task) -> None: logger.info("loaded v2 registry in %.3fs", time.time() - start) self._done.set() - done_task = asyncio.create_task(events._lock.wait_for(start)) + done_task = asyncio.create_task(events._lock.wait_for(await dank_w3.eth.block_number)) done_task.add_done_callback(done_callback) async for _ in events: self._filter_vaults() From 9a53d566a86de0d0dd2f4bd2567cd6d6604e3306 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 43/59] fix: comparison err --- yearn/v2/strategies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yearn/v2/strategies.py b/yearn/v2/strategies.py index d4bb45956..b72b454ba 100644 --- a/yearn/v2/strategies.py +++ b/yearn/v2/strategies.py @@ -93,7 +93,7 @@ def _include_event(self, event: _EventItem) -> bool: def _process_event(self, event: _EventItem) -> int: block = event.block_number logger.debug("%s harvested on %d", self.strategy.name, block) - return event + return block def _unpack_results(views: List[str], results: List[Any], scale: int): From 81f31fa76c540dbec3275353a1a82efca2526b61 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 44/59] fix: attr err --- yearn/v2/strategies.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yearn/v2/strategies.py b/yearn/v2/strategies.py index b72b454ba..b7f4f579c 100644 --- a/yearn/v2/strategies.py +++ b/yearn/v2/strategies.py @@ -88,6 +88,8 @@ def __init__(self, strategy: Strategy): self.strategy = strategy def _include_event(self, event: _EventItem) -> bool: return event.name == "Harvested" + def _get_block_for_obj(self, block: int) -> int: + return block # TODO: work this in somehow: # logger.info("loaded %d harvests %s in %.3fs", len(self._harvests), self.name, time.time() - start) def _process_event(self, event: _EventItem) -> int: From bcded02fee6ec5c927ec051374c3d6f3036058d1 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 45/59] fix: Decimal type errs --- yearn/apy/aero.py | 4 ++-- yearn/apy/curve/rewards.py | 6 +++--- yearn/apy/curve/simple.py | 4 ++-- yearn/apy/velo.py | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/yearn/apy/aero.py b/yearn/apy/aero.py index cce5e30d0..a544ac6b2 100644 --- a/yearn/apy/aero.py +++ b/yearn/apy/aero.py @@ -45,10 +45,10 @@ async def staking(vault: "Vault", staking_rewards: Contract, samples: ApySamples if end < current_time or total_supply == 0 or rate == 0: return Apy("v2:aero_unpopular", gross_apr=0, net_apy=0, fees=fees) - pool_price = await magic.get_price(vault.token.address, block=block, sync=False) + pool_price = float(await magic.get_price(vault.token.address, block=block, sync=False)) reward_token = await staking_rewards.rewardToken.coroutine(block_identifier=block) if hasattr(staking_rewards, "rewardToken") else None token = reward_token - token_price = await magic.get_price(token, block=block, sync=False) + token_price = float(await magic.get_price(token, block=block, sync=False)) gross_apr = (SECONDS_PER_YEAR * (rate / 1e18) * token_price) / (pool_price * (total_supply / 1e18)) diff --git a/yearn/apy/curve/rewards.py b/yearn/apy/curve/rewards.py index 6e002facd..cbee0a6b9 100644 --- a/yearn/apy/curve/rewards.py +++ b/yearn/apy/curve/rewards.py @@ -39,7 +39,7 @@ async def staking(staking_rewards: Contract, pool_price: int, base_asset_price: if token and rate: # Single reward token - token_price = await magic.get_price(token, block=block, sync=False) + token_price = float(await magic.get_price(token, block=block, sync=False)) return (SECONDS_PER_YEAR * (rate / 1e18) * token_price) / ( (pool_price / 1e18) * (total_supply / 1e18) * base_asset_price ) @@ -60,7 +60,7 @@ async def staking(staking_rewards: Contract, pool_price: int, base_asset_price: except ValueError: token = None rate = data.rewardRate / 1e18 if data else 0 - token_price = await magic.get_price(token, block=block, sync=False) or 0 + token_price = float(await magic.get_price(token, block=block, sync=False) or 0) apr += SECONDS_PER_YEAR * rate * token_price / ((pool_price / 1e18) * (total_supply / 1e18) * token_price) queue += 1 try: @@ -88,7 +88,7 @@ async def multi(address: str, pool_price: int, base_asset_price: int, block: Opt token = None if data.periodFinish >= time(): rate = data.rewardRate / 1e18 if data else 0 - token_price = await magic.get_price(token, block=block, sync=False) or 0 + token_price = float(await magic.get_price(token, block=block, sync=False) or 0) apr += SECONDS_PER_YEAR * rate * token_price / ((pool_price / 1e18) * (total_supply / 1e18) * token_price) queue += 1 try: diff --git a/yearn/apy/curve/simple.py b/yearn/apy/curve/simple.py index 3ca75d90b..9a700b215 100644 --- a/yearn/apy/curve/simple.py +++ b/yearn/apy/curve/simple.py @@ -237,7 +237,7 @@ async def calculate_simple(vault, gauge: Gauge, samples: ApySamples) -> Apy: if period_finish < current_time: reward_apr = 0 else: - reward_apr = (SECONDS_PER_YEAR * (rate / 1e18) * token_price) / ((pool_price / 1e18) * (total_supply / 1e18) * base_asset_price) + reward_apr = (SECONDS_PER_YEAR * (rate / 1e18) * float(token_price)) / ((float(pool_price) / 1e18) * (total_supply / 1e18) * float(base_asset_price)) else: reward_apr = 0 @@ -522,7 +522,7 @@ async def _get_reward_apr(self, cvx_strategy, cvx_booster, base_asset_price, poo virtual_rewards_pool.totalSupply.coroutine(), ) - reward_apr = (reward_rate * SECONDS_PER_YEAR * reward_token_price) / (base_asset_price * (pool_price / 1e18) * total_supply) + reward_apr = (reward_rate * SECONDS_PER_YEAR * float(reward_token_price)) / (float(base_asset_price) * (float(pool_price) / 1e18) * total_supply) convex_reward_apr += reward_apr return convex_reward_apr diff --git a/yearn/apy/velo.py b/yearn/apy/velo.py index 4565fa082..c3774630a 100644 --- a/yearn/apy/velo.py +++ b/yearn/apy/velo.py @@ -44,11 +44,11 @@ async def staking(vault: "Vault", staking_rewards: Contract, samples: ApySamples if end < current_time or total_supply == 0 or rate == 0: return Apy("v2:velo_unpopular", gross_apr=0, net_apy=0, fees=fees) - else: - pool_price = await magic.get_price(vault.token.address, block=block, sync=False) - reward_token = await staking_rewards.rewardToken.coroutine(block_identifier=block) if hasattr(staking_rewards, "rewardToken") else None + + pool_price = float(await magic.get_price(vault.token.address, block=block, sync=False)) + reward_token = await staking_rewards.rewardToken.coroutine(block_identifier=block) token = reward_token - token_price = await magic.get_price(token, block=block, sync=False) + token_price = float(await magic.get_price(token, block=block, sync=False)) gross_apr = (SECONDS_PER_YEAR * (rate / 1e18) * token_price) / (pool_price * (total_supply / 1e18)) From b08349d66a6a991b6785febe8622feeb97691e00 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 46/59] fix: deduplicate internal transfers in db --- scripts/exporters/treasury_transactions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/exporters/treasury_transactions.py b/scripts/exporters/treasury_transactions.py index 011896a6a..8c5f48fd8 100644 --- a/scripts/exporters/treasury_transactions.py +++ b/scripts/exporters/treasury_transactions.py @@ -56,9 +56,10 @@ async def load_new_txs(start_block: Block, end_block: Block) -> int: futs = [ asyncio.create_task(insert_treasury_tx(entry)) async for entry in treasury.ledger._get_and_yield(start_block, end_block) - if entry.value + if not isinstance(entry, _Done) and entry.value ] - return sum(await tqdm_asyncio.gather(*futs, desc="Insert Txs to Postgres")) + to_sort = sum(await tqdm_asyncio.gather(*futs, desc="Insert Txs to Postgres")) + return to_sort # NOTE: Things get sketchy when we bump these higher From 4437c4c2b6029f80decfccf8e09c1a15bcb00374 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 47/59] fix: Decimals not json encadable in s3 --- scripts/s3.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/s3.py b/scripts/s3.py index 5cc7640d1..79a4ca31f 100644 --- a/scripts/s3.py +++ b/scripts/s3.py @@ -9,6 +9,7 @@ import warnings from contextlib import suppress from datetime import datetime +from decimal import Decimal from time import time from typing import Union @@ -18,7 +19,6 @@ from brownie import chain from brownie.exceptions import BrownieEnvironmentWarning from telegram.error import BadRequest -from tqdm.asyncio import tqdm_asyncio from y import ERC20, Contract, Network from y.contracts import contract_creation_block_async from y.exceptions import yPriceMagicError @@ -352,3 +352,11 @@ def with_monitoring(): raise error message = f"āœ… {export_mode} Vaults API update for {Network.name()} successful!" updater.bot.send_message(chat_id=private_group, text=message, reply_to_message_id=ping) + +def _dedecimal(dct: dict): + """Decimal type cant be json encoded, we make them into floats""" + for k, v in dct.items(): + if isinstance(v, dict): + _dedecimal(v) + elif isinstance(v, Decimal): + dct[k] = float(v) \ No newline at end of file From 38ea4cdd1bb6a5aceb3a3d036cf061815cada957 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 48/59] str not encodable --- scripts/exporters/transactions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exporters/transactions.py b/scripts/exporters/transactions.py index f6b617a6e..0e3113f59 100644 --- a/scripts/exporters/transactions.py +++ b/scripts/exporters/transactions.py @@ -95,8 +95,8 @@ def process_and_cache_user_txs(last_saved_block=None): from_address=cache_address(row['from']), to_address=cache_address(row['to']), amount = row.amount, - price = price, - value_usd = usd, + price = Decimal(price), + value_usd = Decimal(usd), gas_used = row.gas_used, gas_price = row.gas_price ) From fa9e6b73e019c185f9d4bb3e1ca6c3619938b94a Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 49/59] fix: yeth decimal type err --- yearn/yeth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yearn/yeth.py b/yearn/yeth.py index b8704afd2..036d27e5e 100644 --- a/yearn/yeth.py +++ b/yearn/yeth.py @@ -236,7 +236,7 @@ async def _get_daily_swap_volumes(self, from_block, to_block): volume_in_eth[asset_in] += amount_in * rates[asset_in] volume_out_eth[asset_out] += amount_out * rates[asset_out] - weth_price = await magic.get_price(weth, block=from_block, sync=False) + weth_price = float(await magic.get_price(weth, block=from_block, sync=False)) for i, value in enumerate(volume_in_eth): volume_in_usd[i] = value * weth_price @@ -267,7 +267,7 @@ async def total_value_at(self, block=None): tvls = await asyncio.gather(*[product.total_value_at(block=block) for product in products]) return {product.name: tvl for product, tvl in zip(products, tvls)} - async def active_products_at(self, block=None): + async def active_vaults_at(self, block=None): products = [self.st_yeth] + self.st_yeth.lsts if block: blocks = await asyncio.gather(*[contract_creation_block_async(str(product.address)) for product in products]) From 1472cd17a1edf6a5edb50f5437dcac7699a0976a Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 50/59] fix: active_vaults_at --- yearn/yeth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yearn/yeth.py b/yearn/yeth.py index 036d27e5e..b14bd475f 100644 --- a/yearn/yeth.py +++ b/yearn/yeth.py @@ -258,12 +258,12 @@ async def describe(self, block=None): samples = get_samples() self.swap_volumes = await self._get_daily_swap_volumes(samples.day_ago, samples.now) - products = await self.active_products_at(block) + products = await self.active_vaults_at(block) data = await asyncio.gather(*[product.describe(block=block) for product in products]) return {product.name: desc for product, desc in zip(products, data)} async def total_value_at(self, block=None): - products = await self.active_products_at(block) + products = await self.active_vaults_at(block) tvls = await asyncio.gather(*[product.total_value_at(block=block) for product in products]) return {product.name: tvl for product, tvl in zip(products, tvls)} From 0ba60e69834e0c10631fada8e88f7c962fd50e99 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 51/59] feat: support rkp3r in ypm --- yearn/apy/curve/simple.py | 32 +++----------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/yearn/apy/curve/simple.py b/yearn/apy/curve/simple.py index 9a700b215..26d205e54 100644 --- a/yearn/apy/curve/simple.py +++ b/yearn/apy/curve/simple.py @@ -4,14 +4,12 @@ import os from dataclasses import dataclass from pprint import pformat -from functools import lru_cache from time import time from http import HTTPStatus import requests -from brownie import ZERO_ADDRESS, chain, interface -from dank_mids.brownie_patch import patch_contract +from brownie import ZERO_ADDRESS, chain from eth_abi import encode_single from eth_utils import function_signature_to_4byte_selector as fourbyte from semantic_version import Version @@ -19,7 +17,6 @@ from y.prices import magic from y.prices.stable_swap.curve import curve as y_curve from y.time import get_block_timestamp_async -from y.utils.dank_mids import dank_w3 from yearn import constants from yearn.apy.common import (SECONDS_PER_WEEK, SECONDS_PER_YEAR, Apy, @@ -59,8 +56,6 @@ class Gauge: 'yearn_voter_proxy': '0xF147b8125d2ef93FB6965Db97D6746952a133934', 'convex_voter_proxy': '0x989AEb4d175e16225E39E87d0D97A3360524AD80', 'convex_booster': '0xF403C135812408BFbE8713b5A23a04b3D48AAE31', - 'rkp3r_rewards': '0xEdB67Ee1B171c4eC66E6c10EC43EDBbA20FaE8e9', - 'kp3r': '0x1cEB5cB57C4D4E2b2433641b95Dd330A33185A44', } } @@ -228,7 +223,7 @@ async def calculate_simple(vault, gauge: Gauge, samples: ApySamples) -> Apy: else: reward_data, token_price, total_supply = await asyncio.gather( gauge.gauge.reward_data.coroutine(gauge_reward_token), - _get_reward_token_price(gauge_reward_token), + magic.get_price(gauge_reward_token, block=samples.now, sync=False), gauge.gauge.totalSupply.coroutine(), ) rate = reward_data['rate'] @@ -517,7 +512,7 @@ async def _get_reward_apr(self, cvx_strategy, cvx_booster, base_asset_price, poo if await virtual_rewards_pool.periodFinish.coroutine() > current_time: reward_token = await virtual_rewards_pool.rewardToken.coroutine() reward_token_price, reward_rate, total_supply = await asyncio.gather( - _get_reward_token_price(reward_token, block), + magic.get_price(reward_token, block=block, sync=False), virtual_rewards_pool.rewardRate.coroutine(), virtual_rewards_pool.totalSupply.coroutine(), ) @@ -541,24 +536,3 @@ async def _get_convex_fee(self, cvx_booster, block=None) -> float: def _debt_ratio(self) -> float: """The debt ratio of the Convex strategy.""" return self.vault.vault.strategies(self._cvx_strategy)[2] / 1e4 - - -@lru_cache -def _get_rkp3r() -> Contract: - return patch_contract(interface.rKP3R(addresses[chain.id]['rkp3r_rewards']), dank_w3) - -async def _get_reward_token_price(reward_token, block=None): - if chain.id not in addresses: - return await magic.get_price(reward_token, block=block, sync=False) - - # if the reward token is rKP3R we need to calculate it's price in - # terms of KP3R after the discount - contract_addresses = addresses[chain.id] - if reward_token == contract_addresses['rkp3r_rewards']: - price, discount = await asyncio.gather( - magic.get_price(contract_addresses['kp3r'], block=block, sync=False), - _get_rkp3r().discount.coroutine(block_identifier=block), - ) - return price * (100 - discount) / 100 - else: - return await magic.get_price(reward_token, block=block, sync=False) From 140fcfe0687fcdd65379b66d57f96e06033c7282 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 52/59] fix: s3 --- scripts/s3.py | 45 +++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/scripts/s3.py b/scripts/s3.py index 79a4ca31f..b1c230166 100644 --- a/scripts/s3.py +++ b/scripts/s3.py @@ -10,6 +10,7 @@ from contextlib import suppress from datetime import datetime from decimal import Decimal +from functools import lru_cache from time import time from typing import Union @@ -19,6 +20,7 @@ from brownie import chain from brownie.exceptions import BrownieEnvironmentWarning from telegram.error import BadRequest +from tqdm.asyncio import tqdm_asyncio from y import ERC20, Contract, Network from y.contracts import contract_creation_block_async from y.exceptions import yPriceMagicError @@ -39,15 +41,25 @@ warnings.simplefilter("ignore", BrownieEnvironmentWarning) METRIC_NAME = "yearn.exporter.apy" +DEBUG = os.getenv("DEBUG", None) logs.basicConfig(level=logging.DEBUG) logger = logging.getLogger("yearn.apy") async def wrap_vault( - vault: Union[VaultV1, VaultV2], samples: ApySamples, aliases: dict, icon_url: str, assets_metadata: dict + vault: Union[VaultV1, VaultV2], + samples: ApySamples, + aliases: dict, + icon_url: str, + assets_metadata: dict, + pos: int, + total: int, ) -> dict: - + if DEBUG: + await _get_debug_lock().acquire() + logger.info(f"wrapping vault [{pos}/{total}]: {vault.name} {str(vault.vault)}") + # We don't need results for these right away but they take a while so lets start them now inception_fut = asyncio.create_task(contract_creation_block_async(str(vault.vault))) apy_fut = asyncio.create_task(get_apy(vault, samples)) @@ -72,7 +84,7 @@ async def wrap_vault( if str(vault.vault) in assets_metadata: migration = {"available": assets_metadata[str(vault.vault)][1], "address": assets_metadata[str(vault.vault)][2]} - object = { + data = { "inception": await inception_fut, "address": str(vault.vault), "symbol": vault.symbol if hasattr(vault, "symbol") else await ERC20(vault.vault, asynchronous=True).symbol, @@ -100,9 +112,12 @@ async def wrap_vault( } if chain.id == 1 and any([isinstance(vault, t) for t in [Backscratcher, YveCRVJar]]): - object["special"] = True + data["special"] = True - return object + logger.info(f"done wrapping vault [{pos}/{total}]: {vault.name} {str(vault.vault)}") + if DEBUG: + _get_debug_lock().release() + return data async def get_apy(vault, samples) -> Apy: @@ -225,14 +240,7 @@ async def _main(): assets_metadata = await get_assets_metadata(await registry_v2.vaults) - data = [] - total = len(vaults) - - for i, vault in enumerate(vaults): - pos = i + 1 - logger.info(f"wrapping vault [{pos}/{total}]: {vault.name} {str(vault.vault)}") - data.append(await wrap_vault(vault, samples, aliases, icon_url, assets_metadata)) - logger.info(f"done wrapping vault [{pos}/{total}]: {vault.name} {str(vault.vault)}") + data = await tqdm_asyncio.gather(*[wrap_vault(vault, samples, aliases, icon_url, assets_metadata, i + 1, len(vaults)) for i, vault in enumerate(vaults)]) if len(data) == 0: raise ValueError(f"Data is empty for chain_id: {chain.id}") @@ -263,7 +271,7 @@ def _export(data, file_name, s3_path): with open(file_name, "w+") as f: json.dump(data, f) - if os.getenv("DEBUG", None): + if DEBUG: return for item in _get_s3s(): @@ -326,7 +334,7 @@ def _get_export_paths(suffix): def with_monitoring(): - if os.getenv("DEBUG", None): + if DEBUG: main() return from telegram.ext import Updater @@ -359,4 +367,9 @@ def _dedecimal(dct: dict): if isinstance(v, dict): _dedecimal(v) elif isinstance(v, Decimal): - dct[k] = float(v) \ No newline at end of file + dct[k] = float(v) + +@lru_cache +def _get_debug_lock() -> asyncio.Lock: + # we use this helper function to ensure the lock is always on the right loop + return asyncio.Lock() \ No newline at end of file From 112bb101ea4226f7c78f2313b308ca7f7f029ed8 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 53/59] fix: Decimal not json encodable --- scripts/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/s3.py b/scripts/s3.py index b1c230166..9a97e4ef6 100644 --- a/scripts/s3.py +++ b/scripts/s3.py @@ -117,7 +117,7 @@ async def wrap_vault( logger.info(f"done wrapping vault [{pos}/{total}]: {vault.name} {str(vault.vault)}") if DEBUG: _get_debug_lock().release() - return data + return _dedecimal(data) async def get_apy(vault, samples) -> Apy: From 90e373e372e2b3c95aa931e6e2e54d0e029b2a7b Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 54/59] fix: yeth type err --- yearn/yeth.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/yearn/yeth.py b/yearn/yeth.py index b14bd475f..3085c1805 100644 --- a/yearn/yeth.py +++ b/yearn/yeth.py @@ -49,19 +49,15 @@ def decimals(self): def symbol(self): return 'st-yETH' + async def get_supply(self, block: Optional[Block] = None) -> float: + return (await YETH_POOL.vb_prod_sum.coroutine(block_identifier=block))[1] / 10 ** 18 - async def _get_supply_price(self, block=None): - data = YETH_POOL.vb_prod_sum(block_identifier=block) - supply = data[1] / 1e18 + async def get_price(self, block: Optional[Block] = None) -> Optional[float]: try: - price = await magic.get_price(YETH_TOKEN, block=block, sync=False) + return await magic.get_price(YETH_TOKEN, block=block, sync=False) except yPriceMagicError as e: if not isinstance(e.exception, PriceError): raise e - price = None - - return supply, price - @eth_retry.auto_retry async def apy(self, samples: ApySamples) -> Apy: @@ -83,14 +79,13 @@ async def apy(self, samples: ApySamples) -> Apy: @eth_retry.auto_retry async def tvl(self, block=None) -> Tvl: - supply, price = await self._get_supply_price(block=block) + supply, price = await asyncio.gather(self.get_supply(block), self.get_price(block)) tvl = supply * price if price else None - return Tvl(supply, price, tvl) async def describe(self, block=None): - supply, price = await self._get_supply_price(block=block) + supply, price = await asyncio.gather(self.get_supply(block), self.get_price(block)) try: pool_supply = YETH_POOL.supply(block_identifier=block) total_assets = STAKING_CONTRACT.totalAssets(block_identifier=block) @@ -118,7 +113,7 @@ async def describe(self, block=None): async def total_value_at(self, block=None): - supply, price = await self._get_supply_price(block=block) + supply, price = await asyncio.gather(self.get_supply(block), self.get_price(block)) return supply * price From 01451751ebe0396589fcbfb3d9d098a95714123f Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 55/59] fix: broken import --- yearn/yeth.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/yearn/yeth.py b/yearn/yeth.py index 3085c1805..f1066311c 100644 --- a/yearn/yeth.py +++ b/yearn/yeth.py @@ -1,26 +1,27 @@ import asyncio +import logging import os import re -import logging from datetime import datetime, timezone +from pprint import pformat +from typing import Optional import eth_retry - from brownie import chain -from pprint import pformat - -from y import Contract, magic, Network -from y.time import get_block_timestamp +from y import Contract, Network, magic from y.contracts import contract_creation_block_async +from y.datatypes import Block from y.exceptions import PriceError, yPriceMagicError +from y.time import get_block_timestamp -from yearn.apy.common import (Apy, ApyFees, - ApySamples, SECONDS_PER_YEAR, SECONDS_PER_WEEK, SharePricePoint, calculate_roi, get_samples) +from yearn.apy.common import (SECONDS_PER_WEEK, SECONDS_PER_YEAR, Apy, ApyFees, + ApySamples, SharePricePoint, calculate_roi, + get_samples) from yearn.common import Tvl +from yearn.debug import Debug from yearn.events import decode_logs, get_logs_asap -from yearn.utils import Singleton from yearn.prices.constants import weth -from yearn.debug import Debug +from yearn.utils import Singleton logger = logging.getLogger("yearn.yeth") From f785150347647708ef9c07a26a60e9147440f515 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 56/59] fix: yeth type err --- yearn/yeth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yearn/yeth.py b/yearn/yeth.py index f1066311c..7e0e948d4 100644 --- a/yearn/yeth.py +++ b/yearn/yeth.py @@ -2,7 +2,7 @@ import logging import os import re -from datetime import datetime, timezone +from datetime import datetime from pprint import pformat from typing import Optional @@ -55,7 +55,7 @@ async def get_supply(self, block: Optional[Block] = None) -> float: async def get_price(self, block: Optional[Block] = None) -> Optional[float]: try: - return await magic.get_price(YETH_TOKEN, block=block, sync=False) + return float(await magic.get_price(YETH_TOKEN, block=block, sync=False)) except yPriceMagicError as e: if not isinstance(e.exception, PriceError): raise e From f7d5903e190e0fcda698224ab1561cc9d65cd208 Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 57/59] fix: missing commit --- yearn/treasury/accountant/expenses/people.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/yearn/treasury/accountant/expenses/people.py b/yearn/treasury/accountant/expenses/people.py index 6ba160210..e89f1de04 100644 --- a/yearn/treasury/accountant/expenses/people.py +++ b/yearn/treasury/accountant/expenses/people.py @@ -267,4 +267,7 @@ def is_rantom(tx: TreasuryTx) -> bool: return tx.to_address.address == "0x254b42CaCf7290e72e2C84c0337E36E645784Ce1" def is_tx_creator(tx: TreasuryTx) -> bool: - return tx.to_address.address == "0x4007c53A48DefaB0b9D2F05F34df7bd3088B3299" \ No newline at end of file + return tx.to_address.address == "0x4007c53A48DefaB0b9D2F05F34df7bd3088B3299" + +def is_dinobots(tx: TreasuryTx) -> bool: + return tx.token.symbol == "DAI" and tx._from_nickname == "Yearn yChad Multisig" and tx._to_nickname == "yMechs Multisig" and int(tx.amount) == 47_500 \ No newline at end of file From e9175a9824c95ff1295050a42b4a924fc5b9c25c Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 58/59] fix: type err --- scripts/s3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/s3.py b/scripts/s3.py index 9a97e4ef6..42a945bc0 100644 --- a/scripts/s3.py +++ b/scripts/s3.py @@ -368,6 +368,7 @@ def _dedecimal(dct: dict): _dedecimal(v) elif isinstance(v, Decimal): dct[k] = float(v) + return dct @lru_cache def _get_debug_lock() -> asyncio.Lock: From 6e9ee6d0547dbb90edf3ad24abc6e328c032699b Mon Sep 17 00:00:00 2001 From: BobTheBuidler Date: Mon, 23 Oct 2023 20:11:35 +0000 Subject: [PATCH 59/59] chore: ignore nft from treasury --- yearn/treasury/_setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yearn/treasury/_setup.py b/yearn/treasury/_setup.py index b25195ad7..1d2eb6ed7 100644 --- a/yearn/treasury/_setup.py +++ b/yearn/treasury/_setup.py @@ -25,6 +25,7 @@ "0x528Ff33Bf5bf96B5392c10bc4748d9E9Fb5386B2", # PRM "0x53fFFB19BAcD44b82e204d036D579E86097E5D09", # BGBG "0x57b9d10157f66D8C00a815B5E289a152DeDBE7ed", # ēŽÆēƒč‚” + "0x1d41cf24dF81E3134319BC11c308c5589A486166", # Strangers NFT from @marcoworms <3 }, Network.Arbitrum: { "0x89b0f9dB18FD079063cFA91F174B300C1ce0003C", # AIELON