From 294c98ed5eda6d708769033b95a4d106c90e64e8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Dec 2021 06:55:08 +0100 Subject: [PATCH 1/3] Document exchange.uid part of #6016 --- docs/configuration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index c4689f0a6..00ab66ceb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -126,9 +126,10 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `exchange.uid` | API uid to use for the exchange. Only required when you are in production mode and for exchanges that use uid for API requests.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Supports regex pairs as `.*/BTC`. Not used by VolumePairList. [More information](plugins.md#pairlists-and-pairlist-handlers).
**Datatype:** List | `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting. [More information](plugins.md#pairlists-and-pairlist-handlers).
**Datatype:** List -| `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict +| `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for additional ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation). Please avoid adding exchange secrets here (use the dedicated fields instead), as they may be contained in logs.
**Datatype:** Dict | `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict | `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict | `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded.
*Defaults to `60` minutes.*
**Datatype:** Positive Integer From d3ad4fb52e032f00e58077cd210d59be31fa2f7f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Dec 2021 19:17:00 +0100 Subject: [PATCH 2/3] Don't crash dry-run if orderbook side is empty closes #6018 --- freqtrade/exchange/exchange.py | 24 ++++++++++++++---------- tests/exchange/test_exchange.py | 6 ++++++ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 0ae78cf1b..22041ddef 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -685,16 +685,20 @@ class Exchange: if not self.exchange_has('fetchL2OrderBook'): return True ob = self.fetch_l2_order_book(pair, 1) - if side == 'buy': - price = ob['asks'][0][0] - logger.debug(f"{pair} checking dry buy-order: price={price}, limit={limit}") - if limit >= price: - return True - else: - price = ob['bids'][0][0] - logger.debug(f"{pair} checking dry sell-order: price={price}, limit={limit}") - if limit <= price: - return True + try: + if side == 'buy': + price = ob['asks'][0][0] + logger.debug(f"{pair} checking dry buy-order: price={price}, limit={limit}") + if limit >= price: + return True + else: + price = ob['bids'][0][0] + logger.debug(f"{pair} checking dry sell-order: price={price}, limit={limit}") + if limit <= price: + return True + except IndexError: + # Ignore empty orderbooks when filling - can be filled with the next iteration. + pass return False def check_dry_limit_order_filled(self, order: Dict[str, Any]) -> Dict[str, Any]: diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 5a35675a8..b33e0cbb7 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1026,6 +1026,12 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, assert order_closed['status'] == 'closed' assert order['fee'] + # Empty orderbook test + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', + return_value={'asks': [], 'bids': []}) + exchange._dry_run_open_orders[order['id']]['status'] = 'open' + order_closed = exchange.fetch_dry_run_order(order['id']) + @pytest.mark.parametrize("side,rate,amount,endprice", [ # spread is 25.263-25.266 From ad5c8f601cb8455bcb16d7ae53eefab1f1d88a94 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Dec 2021 20:19:22 +0100 Subject: [PATCH 3/3] Simplify datahandler classes by exploiting commonalities --- freqtrade/data/history/hdf5datahandler.py | 40 ++--------------------- freqtrade/data/history/idatahandler.py | 32 ++++++++++++++++-- freqtrade/data/history/jsondatahandler.py | 24 -------------- 3 files changed, 33 insertions(+), 63 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 1ede3de98..49fac99ea 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -6,7 +6,6 @@ from typing import List, Optional import numpy as np import pandas as pd -from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, ListPairsWithTimeframes, TradeList) @@ -99,19 +98,6 @@ class HDF5DataHandler(IDataHandler): 'low': 'float', 'close': 'float', 'volume': 'float'}) return pairdata - def ohlcv_purge(self, pair: str, timeframe: str) -> bool: - """ - Remove data for this pair - :param pair: Delete data for this pair. - :param timeframe: Timeframe (e.g. "5m") - :return: True when deleted, false if file did not exist. - """ - filename = self._pair_data_filename(self._datadir, pair, timeframe) - if filename.exists(): - filename.unlink() - return True - return False - def ohlcv_append(self, pair: str, timeframe: str, data: pd.DataFrame) -> None: """ Append data to existing data structures @@ -180,17 +166,9 @@ class HDF5DataHandler(IDataHandler): trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None}) return trades.values.tolist() - def trades_purge(self, pair: str) -> bool: - """ - Remove data for this pair - :param pair: Delete data for this pair. - :return: True when deleted, false if file did not exist. - """ - filename = self._pair_trades_filename(self._datadir, pair) - if filename.exists(): - filename.unlink() - return True - return False + @classmethod + def _get_file_extension(cls): + return "h5" @classmethod def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str: @@ -199,15 +177,3 @@ class HDF5DataHandler(IDataHandler): @classmethod def _pair_trades_key(cls, pair: str) -> str: return f"{pair}/trades" - - @classmethod - def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: - pair_s = misc.pair_to_filename(pair) - filename = datadir.joinpath(f'{pair_s}-{timeframe}.h5') - return filename - - @classmethod - def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: - pair_s = misc.pair_to_filename(pair) - filename = datadir.joinpath(f'{pair_s}-trades.h5') - return filename diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 05052b2d7..578d0b5bf 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -12,6 +12,7 @@ from typing import List, Optional, Type from pandas import DataFrame +from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import ListPairsWithTimeframes, TradeList from freqtrade.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe @@ -26,6 +27,13 @@ class IDataHandler(ABC): def __init__(self, datadir: Path) -> None: self._datadir = datadir + @classmethod + def _get_file_extension(cls) -> str: + """ + Get file extension for this particular datahandler + """ + raise NotImplementedError() + @abstractclassmethod def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: """ @@ -70,7 +78,6 @@ class IDataHandler(ABC): :return: DataFrame with ohlcv data, or empty DataFrame """ - @abstractmethod def ohlcv_purge(self, pair: str, timeframe: str) -> bool: """ Remove data for this pair @@ -78,6 +85,11 @@ class IDataHandler(ABC): :param timeframe: Timeframe (e.g. "5m") :return: True when deleted, false if file did not exist. """ + filename = self._pair_data_filename(self._datadir, pair, timeframe) + if filename.exists(): + filename.unlink() + return True + return False @abstractmethod def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: @@ -123,13 +135,17 @@ class IDataHandler(ABC): :return: List of trades """ - @abstractmethod def trades_purge(self, pair: str) -> bool: """ Remove data for this pair :param pair: Delete data for this pair. :return: True when deleted, false if file did not exist. """ + filename = self._pair_trades_filename(self._datadir, pair) + if filename.exists(): + filename.unlink() + return True + return False def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: """ @@ -141,6 +157,18 @@ class IDataHandler(ABC): """ return trades_remove_duplicates(self._trades_load(pair, timerange=timerange)) + @classmethod + def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}') + return filename + + @classmethod + def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') + return filename + def ohlcv_load(self, pair, timeframe: str, timerange: Optional[TimeRange] = None, fill_missing: bool = True, diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 24d6e814b..ccefc8356 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -174,34 +174,10 @@ class JsonDataHandler(IDataHandler): pass return tradesdata - def trades_purge(self, pair: str) -> bool: - """ - Remove data for this pair - :param pair: Delete data for this pair. - :return: True when deleted, false if file did not exist. - """ - filename = self._pair_trades_filename(self._datadir, pair) - if filename.exists(): - filename.unlink() - return True - return False - - @classmethod - def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: - pair_s = misc.pair_to_filename(pair) - filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}') - return filename - @classmethod def _get_file_extension(cls): return "json.gz" if cls._use_zip else "json" - @classmethod - def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: - pair_s = misc.pair_to_filename(pair) - filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') - return filename - class JsonGzDataHandler(JsonDataHandler):