From fe396fffdf9e6747ac417a84c4c0aacd71b9f474 Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 13 Dec 2024 09:28:43 +0900 Subject: [PATCH 01/11] Remove Historic Crypto --- mypy.ini | 4 +- setup.cfg | 2 +- .../abstract_ccxt_pair_converter_plugin.py | 1 + .../plugin/pair_converter/historic_crypto.py | 65 ------------------- ...to.py => test_plugin_coinbase_advanced.py} | 10 +-- 5 files changed, 9 insertions(+), 73 deletions(-) delete mode 100644 src/dali/plugin/pair_converter/historic_crypto.py rename tests/{test_plugin_historic_crypto.py => test_plugin_coinbase_advanced.py} (94%) diff --git a/mypy.ini b/mypy.ini index 6190587a..9e58844a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -91,7 +91,7 @@ disallow_any_expr = False disallow_any_explicit = False disallow_any_expr = False -[mypy-dali.plugin.pair_converter.historic_crypto] +[mypy-dali.plugin.pair_converter.coinbase_advanced] disallow_any_expr = False disallow_any_explicit = False @@ -157,7 +157,7 @@ disallow_any_expr = False disallow_any_explicit = False disallow_any_expr = False -[mypy-test_plugin_historic_crypto] +[mypy-test_plugin_coinbase_advanced] disallow_any_explicit = False disallow_any_expr = False diff --git a/setup.cfg b/setup.cfg index 7071b760..6568f68a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ packages = find: install_requires = backports-datetime-fromisoformat==2.0.1 ccxt==3.0.79 - Historic-Crypto>=0.1.6 + coinbase-advanced-py==1.8.2 jsonschema>=3.2.0 pandas prezzemolo>=0.0.4 diff --git a/src/dali/abstract_ccxt_pair_converter_plugin.py b/src/dali/abstract_ccxt_pair_converter_plugin.py index 2dd4ab0c..805bb415 100644 --- a/src/dali/abstract_ccxt_pair_converter_plugin.py +++ b/src/dali/abstract_ccxt_pair_converter_plugin.py @@ -247,6 +247,7 @@ DAYS_IN_WEEK: int = 7 MANY_YEARS_IN_THE_FUTURE: relativedelta = relativedelta(years=100) + class AssetPairAndHistoricalPrice(NamedTuple): from_asset: str to_asset: str diff --git a/src/dali/plugin/pair_converter/historic_crypto.py b/src/dali/plugin/pair_converter/historic_crypto.py deleted file mode 100644 index 6d3e2f76..00000000 --- a/src/dali/plugin/pair_converter/historic_crypto.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2022 eprbell -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from datetime import datetime, timedelta, timezone -from typing import List, Optional - -from Historic_Crypto import HistoricalData -from rp2.rp2_decimal import RP2Decimal - -from dali.abstract_pair_converter_plugin import AbstractPairConverterPlugin -from dali.historical_bar import HistoricalBar -from dali.transaction_manifest import TransactionManifest - - -class PairConverterPlugin(AbstractPairConverterPlugin): - def name(self) -> str: - return "Historic-Crypto" - - def cache_key(self) -> str: - return self.name() - - def optimize(self, transaction_manifest: TransactionManifest) -> None: - pass - - def get_historic_bar_from_native_source(self, timestamp: datetime, from_asset: str, to_asset: str, exchange: str) -> Optional[HistoricalBar]: - result: Optional[HistoricalBar] = None - time_granularity: List[int] = [60, 300, 900, 3600, 21600, 86400] - # Coinbase API expects UTC timestamps only, see the forum discussion here: - # https://forums.coinbasecloud.dev/t/invalid-end-on-product-candles-endpoint/320 - utc_timestamp = timestamp.astimezone(timezone.utc) - from_timestamp: str = utc_timestamp.strftime("%Y-%m-%d-%H-%M") - retry_count: int = 0 - - while retry_count < len(time_granularity): - try: - seconds = time_granularity[retry_count] - to_timestamp: str = (utc_timestamp + timedelta(seconds=seconds)).strftime("%Y-%m-%d-%H-%M") - historical_data = HistoricalData(f"{from_asset}-{to_asset}", seconds, from_timestamp, to_timestamp, verbose=False).retrieve_data() - historical_data.index = historical_data.index.tz_localize("UTC") # The returned timestamps in the index are timezone naive - historical_data_series = historical_data.reset_index().iloc[0] - result = HistoricalBar( - duration=timedelta(seconds=seconds), - timestamp=historical_data_series.time, - open=RP2Decimal(str(historical_data_series.open)), - high=RP2Decimal(str(historical_data_series.high)), - low=RP2Decimal(str(historical_data_series.low)), - close=RP2Decimal(str(historical_data_series.close)), - volume=RP2Decimal(str(historical_data_series.volume)), - ) - break - except ValueError: - retry_count += 1 - - return result diff --git a/tests/test_plugin_historic_crypto.py b/tests/test_plugin_coinbase_advanced.py similarity index 94% rename from tests/test_plugin_historic_crypto.py rename to tests/test_plugin_coinbase_advanced.py index 4d704ade..0e60ea37 100644 --- a/tests/test_plugin_historic_crypto.py +++ b/tests/test_plugin_coinbase_advanced.py @@ -22,7 +22,7 @@ from dali.cache import CACHE_DIR, load_from_cache from dali.configuration import Keyword from dali.historical_bar import HistoricalBar -from dali.plugin.pair_converter.historic_crypto import PairConverterPlugin +from dali.plugin.pair_converter.coinbase_advanced import PairConverterPlugin BAR_DURATION: timedelta = timedelta(seconds=60) BAR_TIMESTAMP: datetime = datetime(2020, 6, 1, 0, 0).replace(tzinfo=timezone.utc) @@ -53,7 +53,7 @@ def test_historical_prices(self, mocker: Any) -> None: ) # Read price without cache - data = plugin.get_historic_bar_from_native_source(BAR_TIMESTAMP, "BTC", "USD", "Coinbase") + data = plugin.get_historic_bar_from_native_source(BAR_TIMESTAMP, "BTC", "USD", "Coinbase Advanced") assert data assert data.timestamp == BAR_TIMESTAMP @@ -65,7 +65,7 @@ def test_historical_prices(self, mocker: Any) -> None: assert data.volume == BAR_VOLUME # Read price again, but populate plugin cache this time - value = plugin.get_conversion_rate(BAR_TIMESTAMP, "BTC", "USD", "Coinbase") + value = plugin.get_conversion_rate(BAR_TIMESTAMP, "BTC", "USD", "Coinbase Advanced") assert value assert value == BAR_HIGH @@ -74,7 +74,7 @@ def test_historical_prices(self, mocker: Any) -> None: # Load plugin cache and verify cache = load_from_cache(plugin.cache_key()) - key = AssetPairAndTimestamp(BAR_TIMESTAMP, "BTC", "USD", "Coinbase") + key = AssetPairAndTimestamp(BAR_TIMESTAMP, "BTC", "USD", "Coinbase Advanced") assert len(cache) == 1, str(cache) assert key in cache data = cache[key] @@ -94,5 +94,5 @@ def test_missing_historical_prices(self, mocker: Any) -> None: mocker.patch.object(plugin, "get_historic_bar_from_native_source").return_value = None - data = plugin.get_historic_bar_from_native_source(timestamp, "EUR", "JPY", "Coinbase") + data = plugin.get_historic_bar_from_native_source(timestamp, "EUR", "JPY", "Coinbase Advanced") assert data is None From 76aa87ecad6734aebb7be4e2e24f6980bc58a3bb Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 13 Dec 2024 09:29:12 +0900 Subject: [PATCH 02/11] Add Coinbase Stubs --- src/stubs/coinbase/__init__.py | 0 src/stubs/coinbase/rest.pyi | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 src/stubs/coinbase/__init__.py create mode 100644 src/stubs/coinbase/rest.pyi diff --git a/src/stubs/coinbase/__init__.py b/src/stubs/coinbase/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/stubs/coinbase/rest.pyi b/src/stubs/coinbase/rest.pyi new file mode 100644 index 00000000..d43ae1e9 --- /dev/null +++ b/src/stubs/coinbase/rest.pyi @@ -0,0 +1,29 @@ +# Copyright 2024 Neal Chambers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import IO, Any, Optional, Union + +class RESTClient: + def __init__( + self, + api_key: Optional[str] = ..., + api_secret: Optional[str] = ..., + key_file: Optional[Union[IO[bytes], str]] = ..., + base_url: Optional[str] = ..., + timeout: Optional[int] = ..., + verbose: Optional[bool] = ..., + rate_limit_headers: Optional[bool] = ..., + ) -> None: ... + def get_candles(self, product_id: str, start: str, end: str, granularity: str, limit: Optional[int] = None, **kwargs) -> Any: ... # type: ignore + def get_public_candles(self, product_id: str, start: str, end: str, granularity: str, limit: Optional[int] = None, **kwargs) -> Any: ... # type: ignore From 63e5114fc9b14fb8303421a0299ed3d2975c058c Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 13 Dec 2024 09:37:46 +0900 Subject: [PATCH 03/11] Add Coinbase Advanced Pair Converter Plugin --- .../pair_converter/coinbase_advanced.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 src/dali/plugin/pair_converter/coinbase_advanced.py diff --git a/src/dali/plugin/pair_converter/coinbase_advanced.py b/src/dali/plugin/pair_converter/coinbase_advanced.py new file mode 100644 index 00000000..4061a941 --- /dev/null +++ b/src/dali/plugin/pair_converter/coinbase_advanced.py @@ -0,0 +1,93 @@ +# Copyright 2024 orientalperil. Neal Chambers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timedelta, timezone +from typing import Dict, Optional + +from coinbase.rest import RESTClient +from rp2.rp2_decimal import RP2Decimal + +from dali.abstract_pair_converter_plugin import AbstractPairConverterPlugin +from dali.historical_bar import HistoricalBar +from dali.logger import LOGGER +from dali.transaction_manifest import TransactionManifest + +TIME_GRANULARITY: Dict[str, int] = { + "ONE_MINUTE": 60, + "FIVE_MINUTE": 300, + "FIFTEEN_MINUTE": 900, + "THIRTY_MINUTE": 1800, + "ONE_HOUR": 3600, + "TWO_HOUR": 7200, + "SIX_HOUR": 21600, + "ONE_DAY": 86400, +} + + +class PairConverterPlugin(AbstractPairConverterPlugin): + def __init__(self, historical_price_type: str, api_key: Optional[str] = None, api_secret: Optional[str] = None) -> None: + super().__init__(historical_price_type) + self._authorized: bool = False + if api_key is not None and api_secret is not None: + self.client = RESTClient(api_key=api_key, api_secret=api_secret) + self._authorized = True + else: + self.client = RESTClient() + LOGGER.info( + "API key and API secret were not provided for the Coinbase Advanced Pair Converter Plugin. " + "Requests will be throttled. For faster price resolution, please provide a valid " + "API key and secret in the Dali-rp2 configuration file." + ) + + def name(self) -> str: + return "dali_dali_coinbase" + + def cache_key(self) -> str: + return self.name() + + def optimize(self, transaction_manifest: TransactionManifest) -> None: + pass + + def get_historic_bar_from_native_source(self, timestamp: datetime, from_asset: str, to_asset: str, exchange: str) -> Optional[HistoricalBar]: + result: Optional[HistoricalBar] = None + utc_timestamp = timestamp.astimezone(timezone.utc) + start = utc_timestamp.replace(second=0) + end = start + retry_count: int = 0 + + while retry_count < len(TIME_GRANULARITY): + try: + granularity = list(TIME_GRANULARITY.keys())[retry_count] + if self._authorized: + candle = self.client.get_candles(f"{from_asset}-{to_asset}", str(start.timestamp()), str(end.timestamp()), granularity).to_dict()[ + "candles" + ][0] + else: + candle = self.client.get_public_candles(f"{from_asset}-{to_asset}", str(start.timestamp()), str(end.timestamp()), granularity).to_dict()[ + "candles" + ][0] + candle_start = datetime.fromtimestamp(int(candle["start"]), timezone.utc) + result = HistoricalBar( + duration=timedelta(seconds=TIME_GRANULARITY[granularity]), + timestamp=candle_start, + open=RP2Decimal(candle["open"]), + high=RP2Decimal(candle["high"]), + low=RP2Decimal(candle["low"]), + close=RP2Decimal(candle["close"]), + volume=RP2Decimal(candle["volume"]), + ) + except ValueError: + retry_count += 1 + + return result From 37c01617fe503fc52823df2c4879cb300fbbaf83 Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 13 Dec 2024 09:36:19 +0900 Subject: [PATCH 04/11] Update Documentation for Coinbase Advanced Pair Converter Plugin --- docs/configuration_file.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/configuration_file.md b/docs/configuration_file.md index dc038b23..a9ec7f94 100644 --- a/docs/configuration_file.md +++ b/docs/configuration_file.md @@ -623,17 +623,20 @@ Be aware that: * All locked plugins make use of the default forex exchange API, Frankfurter, which provides daily rates from the European Central Bank. Rates for bank holidays and weekends are taken from the previous trading day, so if a rate is requested for Saturday, the Friday rate will be used. -### Historic Crypto -This plugin is based on the Historic_Crypto Python library. +### Coinbase Advanced +This plugin is based on the coinbase_advanced Python library. Initialize this plugin section as follows: 
 [dali.plugin.pair_converter.historic_crypto]
 historical_price_type = <historical_price_type>
+api_key = <api_key>
+api_secret = <api_secret>
Where: * `` is one of `open`, `high`, `low`, `close`, `nearest`. When DaLI downloads historical market data, it captures a `bar` of data surrounding the timestamp of the transaction. Each bar has a starting timestamp, an ending timestamp, and OHLC prices. You can choose which price to select for price lookups. The open, high, low, and close prices are self-explanatory. The `nearest` price is either the open price or the close price of the bar depending on whether the transaction time is nearer the bar starting time or the bar ending time. +* `` and `` can be obtained from your Coinbase account. They are not required, but will allow you many more calls to the API and so will speed up the process of retrieving prices. Without an api key and secret, your calls will be throttled. ## Builtin Sections Builtin sections are used as global configuration of DaLI's behavior. From d672d5132d6ba5fb11e59af4998d22628b73e63d Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 13 Dec 2024 09:45:10 +0900 Subject: [PATCH 05/11] Fix Dead Link --- README.dev.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.dev.md b/README.dev.md index c1466e78..a2fd64ca 100644 --- a/README.dev.md +++ b/README.dev.md @@ -279,7 +279,7 @@ All pair converter plugins are subclasses of [AbstractPairConverterPlugin](src/d * implement the `cache_key()` method; * implement the `get_historic_bar_from_native_source()` method. -For an example of pair converter look at the [Historic-Crypto](src/dali/plugin/pair_converter/historic_crypto.py) plugin. +For an example of pair converter look at the [Coinbase-advanced](src/dali/plugin/pair_converter/coinbase_advanced.py) plugin. ### Country Plugin Development Country plugins are reused from RP2 and their DaLI counterpart has trivial implementation. From 666a71328bf6979a1adaca2939a0b7b4d6049cbe Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Fri, 13 Dec 2024 09:47:25 +0900 Subject: [PATCH 06/11] Replace Historic Crypto in dali_main --- src/dali/dali_main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dali/dali_main.py b/src/dali/dali_main.py index b1686c36..342cfa2c 100644 --- a/src/dali/dali_main.py +++ b/src/dali/dali_main.py @@ -48,8 +48,8 @@ from dali.plugin.pair_converter.ccxt import ( PairConverterPlugin as CcxtPairConverterPlugin, ) -from dali.plugin.pair_converter.historic_crypto import ( - PairConverterPlugin as HistoricCryptoPairConverterPlugin, +from dali.plugin.pair_converter.coinbase_advanced import ( + PairConverterPlugin as CoinbaseAdvancedPairConverterPlugin, ) from dali.transaction_manifest import TransactionManifest from dali.transaction_resolver import resolve_transactions @@ -160,7 +160,7 @@ def _dali_main_internal(country: AbstractCountry) -> None: sys.exit(1) if not pair_converter_list: - pair_converter_list.append(HistoricCryptoPairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value)) + pair_converter_list.append(CoinbaseAdvancedPairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value)) pair_converter_list.append(CcxtPairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value)) LOGGER.info("No pair converter plugins found in configuration file: using default pair converters.") From 1e5afcf149d7c595250d54313149a89ef5b602aa Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Mon, 16 Dec 2024 08:53:33 +0900 Subject: [PATCH 07/11] Rename Plugin --- src/dali/plugin/pair_converter/coinbase_advanced.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dali/plugin/pair_converter/coinbase_advanced.py b/src/dali/plugin/pair_converter/coinbase_advanced.py index 4061a941..0c4f661d 100644 --- a/src/dali/plugin/pair_converter/coinbase_advanced.py +++ b/src/dali/plugin/pair_converter/coinbase_advanced.py @@ -51,7 +51,7 @@ def __init__(self, historical_price_type: str, api_key: Optional[str] = None, ap ) def name(self) -> str: - return "dali_dali_coinbase" + return "coinbase_advanced" def cache_key(self) -> str: return self.name() From 84ce8f161775f0363a15ca24010e19076c2cd4ef Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Mon, 16 Dec 2024 08:54:56 +0900 Subject: [PATCH 08/11] Transform Timestamp to Int --- src/dali/plugin/pair_converter/coinbase_advanced.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dali/plugin/pair_converter/coinbase_advanced.py b/src/dali/plugin/pair_converter/coinbase_advanced.py index 0c4f661d..9a6f2771 100644 --- a/src/dali/plugin/pair_converter/coinbase_advanced.py +++ b/src/dali/plugin/pair_converter/coinbase_advanced.py @@ -70,11 +70,11 @@ def get_historic_bar_from_native_source(self, timestamp: datetime, from_asset: s try: granularity = list(TIME_GRANULARITY.keys())[retry_count] if self._authorized: - candle = self.client.get_candles(f"{from_asset}-{to_asset}", str(start.timestamp()), str(end.timestamp()), granularity).to_dict()[ + candle = self.client.get_candles(f"{from_asset}-{to_asset}", str(int(start.timestamp())), str(int(end.timestamp())), granularity).to_dict()[ "candles" ][0] else: - candle = self.client.get_public_candles(f"{from_asset}-{to_asset}", str(start.timestamp()), str(end.timestamp()), granularity).to_dict()[ + candle = self.client.get_public_candles(f"{from_asset}-{to_asset}", str(int(start.timestamp())), str(int(end.timestamp())), granularity).to_dict()[ "candles" ][0] candle_start = datetime.fromtimestamp(int(candle["start"]), timezone.utc) From a3162bbc6296a88e2d23766ae55eaf4a6f508fcf Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Mon, 16 Dec 2024 10:49:18 +0900 Subject: [PATCH 09/11] Add Test for Granularity --- mypy.ini | 1 + ...ryptoPlugin.test_granularity_response.yaml | 338 ++++++++++++++++++ tests/test_plugin_coinbase_advanced.py | 90 ++++- 3 files changed, 421 insertions(+), 8 deletions(-) create mode 100644 tests/cassettes/test_plugin_coinbase_advanced/TestHistoricCryptoPlugin.test_granularity_response.yaml diff --git a/mypy.ini b/mypy.ini index 9e58844a..aabcb8e7 100644 --- a/mypy.ini +++ b/mypy.ini @@ -160,6 +160,7 @@ disallow_any_expr = False [mypy-test_plugin_coinbase_advanced] disallow_any_explicit = False disallow_any_expr = False +disallow_any_decorated = False [mypy-test_plugin_kraken_csv_download] disallow_any_explicit = False diff --git a/tests/cassettes/test_plugin_coinbase_advanced/TestHistoricCryptoPlugin.test_granularity_response.yaml b/tests/cassettes/test_plugin_coinbase_advanced/TestHistoricCryptoPlugin.test_granularity_response.yaml new file mode 100644 index 00000000..7813a499 --- /dev/null +++ b/tests/cassettes/test_plugin_coinbase_advanced/TestHistoricCryptoPlugin.test_granularity_response.yaml @@ -0,0 +1,338 @@ +interactions: +- request: + body: '{}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2' + Content-Type: + - application/json + User-Agent: + - coinbase-advanced-py/1.8.2 + method: GET + uri: https://api.coinbase.com/api/v3/brokerage/market/products/BTC-USD/candles?end=1591014840&granularity=ONE_MINUTE&start=1591014840 + response: + body: + string: '{"candles":[{"start":"1591014840","low":"9558.02","high":"9566.16","open":"9566.16","close":"9560.73","volume":"7.44296873"}]}' + headers: + CF-Cache-Status: + - EXPIRED + CF-RAY: + - 8f2ae2ee98f58311-KIX + Cache-Control: + - public, max-age=14400 + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 16 Dec 2024 01:26:39 GMT + Expires: + - Mon, 16 Dec 2024 05:26:39 GMT + Last-Modified: + - Mon, 16 Dec 2024 01:26:39 GMT + NEL: + - '{"success_fraction":0.01,"report_to":"cf-nel","max_age":604800}' + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=gFOjApe52Nb1ZcMn6MC1RZ%2BMM16pBPEHwFMnRzQmaB5o6vE7mJtYhpJeQ0%2FOUo6%2B%2FPflJeuaoE1zfwNr5sR1X%2BMhbWp3FSQFGJ%2BhhV3NxcqMZenGs67rdQ44nIL8eVrdWTw%3D"}],"group":"cf-nel","max_age":604800}' + Server: + - cloudflare + Set-Cookie: + - cb_dm=a5f8a8e0-4e50-4ff3-a11f-2993054f0328; Path=/; Domain=coinbase.com; Expires=Sat, + 16 Dec 2034 01:26:39 GMT; HttpOnly; Secure + - __cf_bm=2AfBfhzj6tZmyiK2wEFpVcRJhlrRk7vNxAvTMvlRSq0-1734312399-1.0.1.1-wTWuO8FYyngwG1VQgzakTvSTNjOlR8jCbTNqLcn4bpTf__OKh3iLDGCV_VDkOREEwRpF3Qxg6K6UmGqW8qG.qw; + path=/; expires=Mon, 16-Dec-24 01:56:39 GMT; domain=.coinbase.com; HttpOnly; + Secure; SameSite=None + Transfer-Encoding: + - chunked + access-control-allow-headers: + - Content-Type, Accept, Second-Factor-Proof-Token, Client-Id, Access-Token, + X-Cb-Project-Name, X-Cb-Is-Logged-In, X-Cb-Platform, X-Cb-Session-Uuid, X-Cb-Pagekey, + X-Cb-Ujs, Fingerprint-Tokens, X-Cb-Device-Id, X-Cb-Version-Name + access-control-allow-methods: + - GET,POST,DELETE,PUT + access-control-allow-private-network: + - 'true' + access-control-expose-headers: + - '' + access-control-max-age: + - '7200' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + trace-id: + - '650531074208257991' + - '650531074208257991' + vary: + - Origin, Accept-Encoding + x-content-type-options: + - nosniff + x-dns-prefetch-control: + - 'off' + x-download-options: + - noopen + x-envoy-upstream-service-time: + - '26' + x-frame-options: + - SAMEORIGIN + x-xss-protection: + - 1; mode=block + status: + code: 200 + message: OK +- request: + body: '{}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2' + Content-Type: + - application/json + Cookie: + - cb_dm=a5f8a8e0-4e50-4ff3-a11f-2993054f0328; __cf_bm=2AfBfhzj6tZmyiK2wEFpVcRJhlrRk7vNxAvTMvlRSq0-1734312399-1.0.1.1-wTWuO8FYyngwG1VQgzakTvSTNjOlR8jCbTNqLcn4bpTf__OKh3iLDGCV_VDkOREEwRpF3Qxg6K6UmGqW8qG.qw + User-Agent: + - coinbase-advanced-py/1.8.2 + method: GET + uri: https://api.coinbase.com/api/v3/brokerage/market/products/BTC-USD/candles?end=1591014600&granularity=FIVE_MINUTE&start=1591014600 + response: + body: + string: '{"candles":[{"start":"1591014600","low":"9558.02","high":"9590.45","open":"9574.85","close":"9560.73","volume":"33.9492526"}]}' + headers: + CF-Cache-Status: + - EXPIRED + CF-RAY: + - 8f2ae2f47c838379-KIX + Cache-Control: + - public, max-age=14400 + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 16 Dec 2024 01:26:40 GMT + Expires: + - Mon, 16 Dec 2024 05:26:40 GMT + Last-Modified: + - Mon, 16 Dec 2024 01:26:40 GMT + NEL: + - '{"success_fraction":0.01,"report_to":"cf-nel","max_age":604800}' + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=tpoBYgw3gznUWOSZMLKKbHu6Ex2GF2MpCP7NdJdZCG%2FDDM6owC7pmUMMQteALdwFM8uhWN%2B9or8vFg0iZXQy6R1Vl8W6koEsaUPWymCBGtwry%2BowRDwqwZ1GqmKKlugkaUI%3D"}],"group":"cf-nel","max_age":604800}' + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-headers: + - Content-Type, Accept, Second-Factor-Proof-Token, Client-Id, Access-Token, + X-Cb-Project-Name, X-Cb-Is-Logged-In, X-Cb-Platform, X-Cb-Session-Uuid, X-Cb-Pagekey, + X-Cb-Ujs, Fingerprint-Tokens, X-Cb-Device-Id, X-Cb-Version-Name + access-control-allow-methods: + - GET,POST,DELETE,PUT + access-control-allow-private-network: + - 'true' + access-control-expose-headers: + - '' + access-control-max-age: + - '7200' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + trace-id: + - '7284784732694804126' + - '7284784732694804126' + vary: + - Origin, Accept-Encoding + x-content-type-options: + - nosniff + x-dns-prefetch-control: + - 'off' + x-download-options: + - noopen + x-envoy-upstream-service-time: + - '36' + x-frame-options: + - SAMEORIGIN + x-xss-protection: + - 1; mode=block + status: + code: 200 + message: OK +- request: + body: '{}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2' + Content-Type: + - application/json + Cookie: + - cb_dm=a5f8a8e0-4e50-4ff3-a11f-2993054f0328; __cf_bm=2AfBfhzj6tZmyiK2wEFpVcRJhlrRk7vNxAvTMvlRSq0-1734312399-1.0.1.1-wTWuO8FYyngwG1VQgzakTvSTNjOlR8jCbTNqLcn4bpTf__OKh3iLDGCV_VDkOREEwRpF3Qxg6K6UmGqW8qG.qw + User-Agent: + - coinbase-advanced-py/1.8.2 + method: GET + uri: https://api.coinbase.com/api/v3/brokerage/market/products/BTC-USD/candles?end=1591014600&granularity=FIFTEEN_MINUTE&start=1591014600 + response: + body: + string: '{"candles":[{"start":"1591014600","low":"9558.02","high":"9590.45","open":"9574.85","close":"9569.97","volume":"99.35339347"}]}' + headers: + CF-Cache-Status: + - EXPIRED + CF-RAY: + - 8f2ae2f869e0965a-KIX + Cache-Control: + - public, max-age=14400 + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 16 Dec 2024 01:26:41 GMT + Expires: + - Mon, 16 Dec 2024 05:26:41 GMT + Last-Modified: + - Mon, 16 Dec 2024 01:26:41 GMT + NEL: + - '{"success_fraction":0.01,"report_to":"cf-nel","max_age":604800}' + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=pNBjoqUMS4TsZAlMS7U4AaQpKI3TcoQSv8bRYZ1XsvY9Xua4OH8AXP3n2pbhCHLqsgB8p9eWyzWluNg%2FgmaIUNQluA4Qe6DuNzSwfhoh%2BbRZgjykFPQqqNdgTTVlnQogFaw%3D"}],"group":"cf-nel","max_age":604800}' + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-headers: + - Content-Type, Accept, Second-Factor-Proof-Token, Client-Id, Access-Token, + X-Cb-Project-Name, X-Cb-Is-Logged-In, X-Cb-Platform, X-Cb-Session-Uuid, X-Cb-Pagekey, + X-Cb-Ujs, Fingerprint-Tokens, X-Cb-Device-Id, X-Cb-Version-Name + access-control-allow-methods: + - GET,POST,DELETE,PUT + access-control-allow-private-network: + - 'true' + access-control-expose-headers: + - '' + access-control-max-age: + - '7200' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + trace-id: + - '2922675039930763924' + - '2922675039930763924' + vary: + - Origin, Accept-Encoding + x-content-type-options: + - nosniff + x-dns-prefetch-control: + - 'off' + x-download-options: + - noopen + x-envoy-upstream-service-time: + - '42' + x-frame-options: + - SAMEORIGIN + x-xss-protection: + - 1; mode=block + status: + code: 200 + message: OK +- request: + body: '{}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2' + Content-Type: + - application/json + Cookie: + - cb_dm=a5f8a8e0-4e50-4ff3-a11f-2993054f0328; __cf_bm=2AfBfhzj6tZmyiK2wEFpVcRJhlrRk7vNxAvTMvlRSq0-1734312399-1.0.1.1-wTWuO8FYyngwG1VQgzakTvSTNjOlR8jCbTNqLcn4bpTf__OKh3iLDGCV_VDkOREEwRpF3Qxg6K6UmGqW8qG.qw + User-Agent: + - coinbase-advanced-py/1.8.2 + method: GET + uri: https://api.coinbase.com/api/v3/brokerage/market/products/BTC-USD/candles?end=1590969600&granularity=ONE_DAY&start=1590969600 + response: + body: + string: '{"candles":[{"start":"1590969600","low":"9417.42","high":"10428","open":"9445.83","close":"10208.96","volume":"21676.60828748"}]}' + headers: + CF-Cache-Status: + - EXPIRED + CF-RAY: + - 8f2ae2ffacefd3d9-KIX + Cache-Control: + - public, max-age=14400 + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 16 Dec 2024 01:26:42 GMT + Expires: + - Mon, 16 Dec 2024 05:26:42 GMT + Last-Modified: + - Mon, 16 Dec 2024 01:26:42 GMT + NEL: + - '{"success_fraction":0.01,"report_to":"cf-nel","max_age":604800}' + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=3XVnHJ5TQ8kMiYfjvQ%2Bb%2BWQyDiL2GkMa2HKCA%2Fg3C8Fc6y%2FeStA0c6S4GiPgA%2BSdUNlDYbcaZ%2F6QI5%2F0l72jWf9MxNy%2BsAqjDRvShnSZV%2F7BXmpp%2FTyrmx%2FHVcbPDpsvb8I%3D"}],"group":"cf-nel","max_age":604800}' + Server: + - cloudflare + Transfer-Encoding: + - chunked + access-control-allow-headers: + - Content-Type, Accept, Second-Factor-Proof-Token, Client-Id, Access-Token, + X-Cb-Project-Name, X-Cb-Is-Logged-In, X-Cb-Platform, X-Cb-Session-Uuid, X-Cb-Pagekey, + X-Cb-Ujs, Fingerprint-Tokens, X-Cb-Device-Id, X-Cb-Version-Name + access-control-allow-methods: + - GET,POST,DELETE,PUT + access-control-allow-private-network: + - 'true' + access-control-expose-headers: + - '' + access-control-max-age: + - '7200' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + trace-id: + - '8313227568894027626' + - '8313227568894027626' + vary: + - Origin, Accept-Encoding + x-content-type-options: + - nosniff + x-dns-prefetch-control: + - 'off' + x-download-options: + - noopen + x-envoy-upstream-service-time: + - '29' + x-frame-options: + - SAMEORIGIN + x-xss-protection: + - 1; mode=block + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_plugin_coinbase_advanced.py b/tests/test_plugin_coinbase_advanced.py index 0e60ea37..0b5fe059 100644 --- a/tests/test_plugin_coinbase_advanced.py +++ b/tests/test_plugin_coinbase_advanced.py @@ -16,6 +16,7 @@ from datetime import datetime, timedelta, timezone from typing import Any +import pytest from rp2.rp2_decimal import RP2Decimal from dali.abstract_pair_converter_plugin import AssetPairAndTimestamp @@ -25,12 +26,36 @@ from dali.plugin.pair_converter.coinbase_advanced import PairConverterPlugin BAR_DURATION: timedelta = timedelta(seconds=60) -BAR_TIMESTAMP: datetime = datetime(2020, 6, 1, 0, 0).replace(tzinfo=timezone.utc) -BAR_LOW: RP2Decimal = RP2Decimal("9430.01") -BAR_HIGH: RP2Decimal = RP2Decimal("9447.52") -BAR_OPEN: RP2Decimal = RP2Decimal("9445.83") -BAR_CLOSE: RP2Decimal = RP2Decimal("9435.80") -BAR_VOLUME: RP2Decimal = RP2Decimal("1") +BAR_TIMESTAMP: datetime = datetime(2020, 6, 1, 12, 34).replace(tzinfo=timezone.utc) +BAR_LOW: RP2Decimal = RP2Decimal("9558.02") +BAR_HIGH: RP2Decimal = RP2Decimal("9566.16") +BAR_OPEN: RP2Decimal = RP2Decimal("9566.16") +BAR_CLOSE: RP2Decimal = RP2Decimal("9560.73") +BAR_VOLUME: RP2Decimal = RP2Decimal("7.44296873") + +FIVE_MINUTE_DURATION: timedelta = timedelta(seconds=300) +FIVE_MINUTE_TIMESTAMP: datetime = datetime(2020, 6, 1, 12, 30).replace(tzinfo=timezone.utc) +FIVE_MINUTE_LOW: RP2Decimal = RP2Decimal("9558.02") +FIVE_MINUTE_HIGH: RP2Decimal = RP2Decimal("9590.45") +FIVE_MINUTE_OPEN: RP2Decimal = RP2Decimal("9574.85") +FIVE_MINUTE_CLOSE: RP2Decimal = RP2Decimal("9560.73") +FIVE_MINUTE_VOLUME: RP2Decimal = RP2Decimal("33.9492526") + +FIFTEEN_MINUTE_DURATION: timedelta = timedelta(seconds=900) +FIFTEEN_MINUTE_TIMESTAMP: datetime = datetime(2020, 6, 1, 12, 30, tzinfo=timezone.utc) +FIFTEEN_MINUTE_OPEN: RP2Decimal = RP2Decimal("9574.85") +FIFTEEN_MINUTE_HIGH: RP2Decimal = RP2Decimal("9590.45") +FIFTEEN_MINUTE_LOW: RP2Decimal = RP2Decimal("9558.02") +FIFTEEN_MINUTE_CLOSE: RP2Decimal = RP2Decimal("9569.97") +FIFTEEN_MINUTE_VOLUME: RP2Decimal = RP2Decimal("99.35339347") + +ONE_DAY_DURATION: timedelta = timedelta(days=1) +ONE_DAY_TIMESTAMP: datetime = datetime(2020, 6, 1, 0, 0, tzinfo=timezone.utc) +ONE_DAY_OPEN: RP2Decimal = RP2Decimal("9445.83") +ONE_DAY_HIGH: RP2Decimal = RP2Decimal("10428") +ONE_DAY_LOW: RP2Decimal = RP2Decimal("9417.42") +ONE_DAY_CLOSE: RP2Decimal = RP2Decimal("10208.96") +ONE_DAY_VOLUME: RP2Decimal = RP2Decimal("21676.60828748") class TestHistoricCryptoPlugin: @@ -57,7 +82,6 @@ def test_historical_prices(self, mocker: Any) -> None: assert data assert data.timestamp == BAR_TIMESTAMP - assert data.timestamp == BAR_TIMESTAMP assert data.low == BAR_LOW assert data.high == BAR_HIGH assert data.open == BAR_OPEN @@ -81,7 +105,6 @@ def test_historical_prices(self, mocker: Any) -> None: assert data assert data.timestamp == BAR_TIMESTAMP - assert data.timestamp == BAR_TIMESTAMP assert data.low == BAR_LOW assert data.high == BAR_HIGH assert data.open == BAR_OPEN @@ -96,3 +119,54 @@ def test_missing_historical_prices(self, mocker: Any) -> None: data = plugin.get_historic_bar_from_native_source(timestamp, "EUR", "JPY", "Coinbase Advanced") assert data is None + + @pytest.mark.vcr + def test_granularity_response(self, mocker: Any) -> None: + plugin = PairConverterPlugin(Keyword.HISTORICAL_PRICE_HIGH.value) + timestamp = datetime(2020, 6, 1, 12, 34).replace(tzinfo=timezone.utc) + + data = plugin.get_historic_bar_from_native_source(timestamp, "BTC", "USD", "Coinbase Advanced") + assert data + assert data.timestamp == BAR_TIMESTAMP + assert data.low == BAR_LOW + assert data.high == BAR_HIGH + assert data.open == BAR_OPEN + assert data.close == BAR_CLOSE + assert data.volume == BAR_VOLUME + assert data.duration == BAR_DURATION + + mocker.patch("dali.plugin.pair_converter.coinbase_advanced.TIME_GRANULARITY", {"FIVE_MINUTE": 300}) + + data = plugin.get_historic_bar_from_native_source(timestamp, "BTC", "USD", "Coinbase Advanced") + assert data + assert data.timestamp == FIVE_MINUTE_TIMESTAMP + assert data.low == FIVE_MINUTE_LOW + assert data.high == FIVE_MINUTE_HIGH + assert data.open == FIVE_MINUTE_OPEN + assert data.close == FIVE_MINUTE_CLOSE + assert data.volume == FIVE_MINUTE_VOLUME + assert data.duration == FIVE_MINUTE_DURATION + + mocker.patch("dali.plugin.pair_converter.coinbase_advanced.TIME_GRANULARITY", {"FIFTEEN_MINUTE": 900}) + + data = plugin.get_historic_bar_from_native_source(timestamp, "BTC", "USD", "Coinbase Advanced") + assert data + assert data.timestamp == FIFTEEN_MINUTE_TIMESTAMP + assert data.low == FIFTEEN_MINUTE_LOW + assert data.high == FIFTEEN_MINUTE_HIGH + assert data.open == FIFTEEN_MINUTE_OPEN + assert data.close == FIFTEEN_MINUTE_CLOSE + assert data.volume == FIFTEEN_MINUTE_VOLUME + assert data.duration == FIFTEEN_MINUTE_DURATION + + mocker.patch("dali.plugin.pair_converter.coinbase_advanced.TIME_GRANULARITY", {"ONE_DAY": 86400}) + + data = plugin.get_historic_bar_from_native_source(timestamp, "BTC", "USD", "Coinbase Advanced") + assert data + assert data.timestamp == ONE_DAY_TIMESTAMP + assert data.low == ONE_DAY_LOW + assert data.high == ONE_DAY_HIGH + assert data.open == ONE_DAY_OPEN + assert data.close == ONE_DAY_CLOSE + assert data.volume == ONE_DAY_VOLUME + assert data.duration == ONE_DAY_DURATION From e215facb6a09ec1479cb9263b408d82c8e4e63f7 Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Mon, 16 Dec 2024 10:31:54 +0900 Subject: [PATCH 10/11] Adjust Start Time to Match Granularity --- .../plugin/pair_converter/coinbase_advanced.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/dali/plugin/pair_converter/coinbase_advanced.py b/src/dali/plugin/pair_converter/coinbase_advanced.py index 9a6f2771..11f80fde 100644 --- a/src/dali/plugin/pair_converter/coinbase_advanced.py +++ b/src/dali/plugin/pair_converter/coinbase_advanced.py @@ -62,21 +62,19 @@ def optimize(self, transaction_manifest: TransactionManifest) -> None: def get_historic_bar_from_native_source(self, timestamp: datetime, from_asset: str, to_asset: str, exchange: str) -> Optional[HistoricalBar]: result: Optional[HistoricalBar] = None utc_timestamp = timestamp.astimezone(timezone.utc) - start = utc_timestamp.replace(second=0) - end = start + start = utc_timestamp retry_count: int = 0 while retry_count < len(TIME_GRANULARITY): try: granularity = list(TIME_GRANULARITY.keys())[retry_count] + start_epoch = int(start.timestamp()) + start_epoch = start_epoch - (start_epoch % TIME_GRANULARITY[granularity]) + end_epoch = start_epoch if self._authorized: - candle = self.client.get_candles(f"{from_asset}-{to_asset}", str(int(start.timestamp())), str(int(end.timestamp())), granularity).to_dict()[ - "candles" - ][0] + candle = self.client.get_candles(f"{from_asset}-{to_asset}", str(start_epoch), str(end_epoch), granularity).to_dict()["candles"][0] else: - candle = self.client.get_public_candles(f"{from_asset}-{to_asset}", str(int(start.timestamp())), str(int(end.timestamp())), granularity).to_dict()[ - "candles" - ][0] + candle = self.client.get_public_candles(f"{from_asset}-{to_asset}", str(start_epoch), str(end_epoch), granularity).to_dict()["candles"][0] candle_start = datetime.fromtimestamp(int(candle["start"]), timezone.utc) result = HistoricalBar( duration=timedelta(seconds=TIME_GRANULARITY[granularity]), @@ -89,5 +87,8 @@ def get_historic_bar_from_native_source(self, timestamp: datetime, from_asset: s ) except ValueError: retry_count += 1 + if result: + break + retry_count += 1 return result From 433902e6672401577a308fcf29b932e94a49fe44 Mon Sep 17 00:00:00 2001 From: Neal Chambers Date: Mon, 16 Dec 2024 10:36:56 +0900 Subject: [PATCH 11/11] Remove Gzip Encoding --- ...estHistoricCryptoPlugin.test_granularity_response.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/cassettes/test_plugin_coinbase_advanced/TestHistoricCryptoPlugin.test_granularity_response.yaml b/tests/cassettes/test_plugin_coinbase_advanced/TestHistoricCryptoPlugin.test_granularity_response.yaml index 7813a499..059085df 100644 --- a/tests/cassettes/test_plugin_coinbase_advanced/TestHistoricCryptoPlugin.test_granularity_response.yaml +++ b/tests/cassettes/test_plugin_coinbase_advanced/TestHistoricCryptoPlugin.test_granularity_response.yaml @@ -28,8 +28,6 @@ interactions: - public, max-age=14400 Connection: - keep-alive - Content-Encoding: - - gzip Content-Type: - application/json; charset=utf-8 Date: @@ -117,8 +115,6 @@ interactions: - public, max-age=14400 Connection: - keep-alive - Content-Encoding: - - gzip Content-Type: - application/json; charset=utf-8 Date: @@ -200,8 +196,6 @@ interactions: - public, max-age=14400 Connection: - keep-alive - Content-Encoding: - - gzip Content-Type: - application/json; charset=utf-8 Date: @@ -283,8 +277,6 @@ interactions: - public, max-age=14400 Connection: - keep-alive - Content-Encoding: - - gzip Content-Type: - application/json; charset=utf-8 Date: