From b1dbc3a65f3bdab8b1f2ad7fcc127f7de1d42569 Mon Sep 17 00:00:00 2001 From: Wagner Costa Santos Date: Thu, 22 Sep 2022 12:13:51 -0300 Subject: [PATCH 1/9] remove function remove_training_from_backtesting and ensure BT period is correct with startup_candle_count --- freqtrade/data/dataprovider.py | 12 +++++++++--- freqtrade/freqai/data_kitchen.py | 23 ----------------------- freqtrade/optimize/backtesting.py | 9 +++++++-- 3 files changed, 16 insertions(+), 28 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 43850ddd9..51cacc2c5 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -90,8 +90,12 @@ class DataProvider: if saved_pair not in self.__cached_pairs_backtesting: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) + # It is not necessary to add the training candles, as they + # were already added at the beginning of the backtest. + add_train_candles = False + # Move informative start time respecting startup_candle_count - startup_candles = self.get_required_startup(str(timeframe)) + startup_candles = self.get_required_startup(str(timeframe), add_train_candles) tf_seconds = timeframe_to_seconds(str(timeframe)) timerange.subtract_start(tf_seconds * startup_candles) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( @@ -105,7 +109,7 @@ class DataProvider: ) return self.__cached_pairs_backtesting[saved_pair].copy() - def get_required_startup(self, timeframe: str) -> int: + def get_required_startup(self, timeframe: str, add_train_candles: bool = True) -> int: freqai_config = self._config.get('freqai', {}) if not freqai_config.get('enabled', False): return self._config.get('startup_candle_count', 0) @@ -115,7 +119,9 @@ class DataProvider: # make sure the startupcandles is at least the set maximum indicator periods self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods)) tf_seconds = timeframe_to_seconds(timeframe) - train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds + train_candles = 0 + if add_train_candles: + train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds total_candles = int(self._config['startup_candle_count'] + train_candles) logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') return total_candles diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index d2abd0ad2..dd2ef3bad 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -466,27 +466,6 @@ class FreqaiDataKitchen: return df - def remove_training_from_backtesting( - self - ) -> DataFrame: - """ - Function which takes the backtesting time range and - remove training data from dataframe, keeping only the - startup_candle_count candles - """ - startup_candle_count = self.config.get('startup_candle_count', 0) - tf = self.config['timeframe'] - tr = self.config["timerange"] - - backtesting_timerange = TimeRange.parse_timerange(tr) - if startup_candle_count > 0 and backtesting_timerange: - backtesting_timerange.subtract_start(timeframe_to_seconds(tf) * startup_candle_count) - - start = datetime.fromtimestamp(backtesting_timerange.startts, tz=timezone.utc) - df = self.return_dataframe - df = df.loc[df["date"] >= start, :] - return df - def principal_component_analysis(self) -> None: """ Performs Principal Component Analysis on the data for dimensionality reduction @@ -979,8 +958,6 @@ class FreqaiDataKitchen: to_keep = [col for col in dataframe.columns if not col.startswith("&")] self.return_dataframe = pd.concat([dataframe[to_keep], self.full_df], axis=1) - - self.return_dataframe = self.remove_training_from_backtesting() self.full_df = DataFrame() return diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0a05d740d..20429eb97 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -139,9 +139,14 @@ class Backtesting: # Get maximum required startup period self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) + self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) + + if self.config.get('freqai', {}).get('enabled', False): + # For FreqAI, increase the required_startup to includes the training data + self.required_startup = self.dataprovider.get_required_startup(self.timeframe) + # Add maximum startup candle count to configuration for informative pairs support self.config['startup_candle_count'] = self.required_startup - self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) self.trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT) # strategies which define "can_short=True" will fail to load in Spot mode. @@ -217,7 +222,7 @@ class Backtesting: pairs=self.pairlists.whitelist, timeframe=self.timeframe, timerange=self.timerange, - startup_candles=self.dataprovider.get_required_startup(self.timeframe), + startup_candles=self.config['startup_candle_count'], fail_without_data=True, data_format=self.config.get('dataformat_ohlcv', 'json'), candle_type=self.config.get('candle_type_def', CandleType.SPOT) From 00b192b4dfb8b14379ebaac0508fc2be9c8cd718 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Sep 2022 14:51:58 +0200 Subject: [PATCH 2/9] Add test to verify #7449 --- tests/exchange/test_exchange.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 71690ecdf..b91046b75 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -20,6 +20,7 @@ from freqtrade.exchange import (Binance, Bittrex, Exchange, Kraken, amount_to_pr timeframe_to_prev_date, timeframe_to_seconds) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, calculate_backoff, remove_credentials) +from freqtrade.exchange.exchange import amount_to_contract_precision from freqtrade.resolvers.exchange_resolver import ExchangeResolver from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re, num_log_has_re @@ -4470,6 +4471,7 @@ def test__amount_to_contracts( ('ADA/USDT:USDT', 10.4445555, 10.4, 10.444), ('LTC/ETH', 30, 30, 30), ('LTC/USD', 30, 30, 30), + ('ADA/USDT:USDT', 1.17, 1.1, 1.17), # contract size of 10 ('ETH/USDT:USDT', 10.111, 10.1, 10), ('ETH/USDT:USDT', 10.188, 10.1, 10), From 4efe2e9bc48746ce871a751e7a3069a6f50106d9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Sep 2022 14:55:58 +0200 Subject: [PATCH 3/9] use FtPrecise to convert to contracts and back --- freqtrade/exchange/exchange.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index c68fc5873..f01e464fa 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2891,7 +2891,7 @@ def amount_to_contracts(amount: float, contract_size: Optional[float]) -> float: :return: num-contracts """ if contract_size and contract_size != 1: - return amount / contract_size + return float(FtPrecise(amount) / FtPrecise(contract_size)) else: return amount @@ -2905,7 +2905,7 @@ def contracts_to_amount(num_contracts: float, contract_size: Optional[float]) -> """ if contract_size and contract_size != 1: - return num_contracts * contract_size + return float(FtPrecise(num_contracts) * FtPrecise(contract_size)) else: return num_contracts From 98ba57ffaa99cde45d24106354edaeddf4d72525 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Sep 2022 15:25:04 +0200 Subject: [PATCH 4/9] Better test for contract calculation change closes #7449 --- tests/exchange/test_exchange.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index b91046b75..37ba2ca97 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -4499,6 +4499,20 @@ def test_amount_to_contract_precision( assert result_size == expected_fut +@pytest.mark.parametrize('amount,precision,precision_mode,contract_size,expected', [ + (1.17, 1.0, 4, 0.01, 1.17), # Tick size + (1.17, 1.0, 2, 0.01, 1.17), # + (1.16, 1.0, 4, 0.01, 1.16), # + (1.16, 1.0, 2, 0.01, 1.16), # + (1.13, 1.0, 2, 0.01, 1.13), # + (10.988, 1.0, 2, 10, 10), + (10.988, 1.0, 4, 10, 10), +]) +def test_amount_to_contract_precision2(amount, precision, precision_mode, contract_size, expected): + res = amount_to_contract_precision(amount, precision, precision_mode, contract_size) + assert pytest.approx(res) == expected + + @pytest.mark.parametrize('exchange_name,open_rate,is_short,trading_mode,margin_mode', [ # Bittrex ('bittrex', 2.0, False, 'spot', None), From 166ae8e3a1cd7c2169dd41a534f409dba7e0845e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Sep 2022 15:51:20 +0200 Subject: [PATCH 5/9] Remove missleading comment --- freqtrade/data/dataprovider.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 51cacc2c5..ac3b61d1d 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -90,12 +90,10 @@ class DataProvider: if saved_pair not in self.__cached_pairs_backtesting: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) + # It is not necessary to add the training candles, as they # were already added at the beginning of the backtest. - add_train_candles = False - - # Move informative start time respecting startup_candle_count - startup_candles = self.get_required_startup(str(timeframe), add_train_candles) + startup_candles = self.get_required_startup(str(timeframe), False) tf_seconds = timeframe_to_seconds(str(timeframe)) timerange.subtract_start(tf_seconds * startup_candles) self.__cached_pairs_backtesting[saved_pair] = load_pair_history( From 53c8e0923fe2fc2d0048f004c89a452057c01a8b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Sep 2022 16:10:42 +0200 Subject: [PATCH 6/9] Improve typing in message_consumer --- freqtrade/rpc/external_message_consumer.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index 99ba39f76..bf71c24ea 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -8,7 +8,7 @@ import asyncio import logging import socket from threading import Thread -from typing import TYPE_CHECKING, Any, Callable, Dict, List +from typing import TYPE_CHECKING, Any, Callable, Dict, List, TypedDict import websockets from pydantic import ValidationError @@ -29,6 +29,13 @@ if TYPE_CHECKING: import websockets.exceptions +class Producer(TypedDict): + name: str + host: str + port: int + ws_token: str + + logger = logging.getLogger(__name__) @@ -55,7 +62,7 @@ class ExternalMessageConsumer: self._emc_config = self._config.get('external_message_consumer', {}) self.enabled = self._emc_config.get('enabled', False) - self.producers = self._emc_config.get('producers', []) + self.producers: List[Producer] = self._emc_config.get('producers', []) self.wait_timeout = self._emc_config.get('wait_timeout', 300) # in seconds self.ping_timeout = self._emc_config.get('ping_timeout', 10) # in seconds @@ -162,7 +169,7 @@ class ExternalMessageConsumer: # Stop the loop once we are done self._loop.stop() - async def _handle_producer_connection(self, producer: Dict[str, Any], lock: asyncio.Lock): + async def _handle_producer_connection(self, producer: Producer, lock: asyncio.Lock): """ Main connection loop for the consumer @@ -175,7 +182,7 @@ class ExternalMessageConsumer: # Exit silently pass - async def _create_connection(self, producer: Dict[str, Any], lock: asyncio.Lock): + async def _create_connection(self, producer: Producer, lock: asyncio.Lock): """ Actually creates and handles the websocket connection, pinging on timeout and handling connection errors. @@ -236,7 +243,7 @@ class ExternalMessageConsumer: async def _receive_messages( self, channel: WebSocketChannel, - producer: Dict[str, Any], + producer: Producer, lock: asyncio.Lock ): """ @@ -277,7 +284,7 @@ class ExternalMessageConsumer: break - def handle_producer_message(self, producer: Dict[str, Any], message: Dict[str, Any]): + def handle_producer_message(self, producer: Producer, message: Dict[str, Any]): """ Handles external messages from a Producer """ From 50dfde7048e4ed36e6b5e0ad62f0f59ca1dd6611 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Sep 2022 16:11:15 +0200 Subject: [PATCH 7/9] Remove unnecessary typing import --- freqtrade/rpc/external_message_consumer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index bf71c24ea..bb96279d9 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -26,7 +26,6 @@ from freqtrade.rpc.api_server.ws_schemas import (WSAnalyzedDFMessage, WSAnalyzed if TYPE_CHECKING: import websockets.connect - import websockets.exceptions class Producer(TypedDict): From 8d77ba118c60ef062a5874197022be8d35ac819f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Sep 2022 16:15:15 +0200 Subject: [PATCH 8/9] Fix line endings --- config_examples/config_freqai.example.json | 2 +- docker/Dockerfile.freqai | 1 - freqtrade/exchange/binance_leverage_tiers.json | 2 +- setup.cfg | 1 - 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index 3a8a3b273..fe5e35c1d 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -94,4 +94,4 @@ "internals": { "process_throttle_secs": 5 } -} \ No newline at end of file +} diff --git a/docker/Dockerfile.freqai b/docker/Dockerfile.freqai index 9a2f75700..e9f04f3d6 100644 --- a/docker/Dockerfile.freqai +++ b/docker/Dockerfile.freqai @@ -6,4 +6,3 @@ FROM ${sourceimage}:${sourcetag} COPY requirements-freqai.txt /freqtrade/ RUN pip install -r requirements-freqai.txt --user --no-cache-dir - diff --git a/freqtrade/exchange/binance_leverage_tiers.json b/freqtrade/exchange/binance_leverage_tiers.json index 2fa326bb1..c3b86684b 100644 --- a/freqtrade/exchange/binance_leverage_tiers.json +++ b/freqtrade/exchange/binance_leverage_tiers.json @@ -19209,4 +19209,4 @@ } } ] -} \ No newline at end of file +} diff --git a/setup.cfg b/setup.cfg index d711534d9..60ec8a75f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,4 +49,3 @@ exclude = __pycache__, .eggs, user_data, - From 873eb5f2cad30eae190c3949d24165a8ddb29f6b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Sep 2022 16:38:56 +0200 Subject: [PATCH 9/9] Improve EMC config validations --- freqtrade/configuration/config_validation.py | 19 +++++++ freqtrade/rpc/external_message_consumer.py | 15 ------ tests/rpc/test_rpc_emc.py | 14 +----- tests/test_configuration.py | 52 ++++++++++++++++++++ 4 files changed, 72 insertions(+), 28 deletions(-) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 8d9112bef..7055d9551 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -1,4 +1,5 @@ import logging +from collections import Counter from copy import deepcopy from typing import Any, Dict @@ -85,6 +86,7 @@ def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False) _validate_unlimited_amount(conf) _validate_ask_orderbook(conf) _validate_freqai_hyperopt(conf) + _validate_consumers(conf) validate_migrated_strategy_settings(conf) # validate configuration before returning @@ -332,6 +334,23 @@ def _validate_freqai_hyperopt(conf: Dict[str, Any]) -> None: 'Using analyze-per-epoch parameter is not supported with a FreqAI strategy.') +def _validate_consumers(conf: Dict[str, Any]) -> None: + emc_conf = conf.get('external_message_consumer', {}) + if emc_conf.get('enabled', False): + if len(emc_conf.get('producers', [])) < 1: + raise OperationalException("You must specify at least 1 Producer to connect to.") + + producer_names = [p['name'] for p in emc_conf.get('producers', [])] + duplicates = [item for item, count in Counter(producer_names).items() if count > 1] + if duplicates: + raise OperationalException( + f"Producer names must be unique. Duplicate: {', '.join(duplicates)}") + if conf.get('process_only_new_candles', True): + # Warning here or require it? + logger.warning("To receive best performance with external data, " + "please set `process_only_new_candles` to False") + + def _strategy_settings(conf: Dict[str, Any]) -> None: process_deprecated_setting(conf, None, 'use_sell_signal', None, 'use_exit_signal') diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index bb96279d9..dcfe1d109 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -15,7 +15,6 @@ from pydantic import ValidationError from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import RPCMessageType -from freqtrade.exceptions import OperationalException from freqtrade.misc import remove_entry_exit_signals from freqtrade.rpc.api_server.ws import WebSocketChannel from freqtrade.rpc.api_server.ws_schemas import (WSAnalyzedDFMessage, WSAnalyzedDFRequest, @@ -74,8 +73,6 @@ class ExternalMessageConsumer: # as the websockets client expects bytes. self.message_size_limit = (self._emc_config.get('message_size_limit', 8) << 20) - self.validate_config() - # Setting these explicitly as they probably shouldn't be changed by a user # Unless we somehow integrate this with the strategy to allow creating # callbacks for the messages @@ -96,18 +93,6 @@ class ExternalMessageConsumer: self.start() - def validate_config(self): - """ - Make sure values are what they are supposed to be - """ - if self.enabled and len(self.producers) < 1: - raise OperationalException("You must specify at least 1 Producer to connect to.") - - if self.enabled and self._config.get('process_only_new_candles', True): - # Warning here or require it? - logger.warning("To receive best performance with external data, " - "please set `process_only_new_candles` to False") - def start(self): """ Start the main internal loop in another thread to run coroutines diff --git a/tests/rpc/test_rpc_emc.py b/tests/rpc/test_rpc_emc.py index 41faaf249..2649c5460 100644 --- a/tests/rpc/test_rpc_emc.py +++ b/tests/rpc/test_rpc_emc.py @@ -11,7 +11,6 @@ import pytest import websockets from freqtrade.data.dataprovider import DataProvider -from freqtrade.exceptions import OperationalException from freqtrade.rpc.external_message_consumer import ExternalMessageConsumer from tests.conftest import log_has, log_has_re, log_has_when @@ -73,23 +72,12 @@ def test_emc_shutdown(patched_emc, caplog): assert not log_has("Stopping ExternalMessageConsumer", caplog) -def test_emc_init(patched_emc, default_conf): +def test_emc_init(patched_emc): # Test the settings were set correctly assert patched_emc.initial_candle_limit <= 1500 assert patched_emc.wait_timeout > 0 assert patched_emc.sleep_time > 0 - default_conf.update({ - "external_message_consumer": { - "enabled": True, - "producers": [] - } - }) - dataprovider = DataProvider(default_conf, None, None, None) - with pytest.raises(OperationalException, - match="You must specify at least 1 Producer to connect to."): - ExternalMessageConsumer(default_conf, dataprovider) - # Parametrize this? def test_emc_handle_producer_message(patched_emc, caplog, ohlcv_history): diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 2825ede5c..99edf0233 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1089,6 +1089,58 @@ def test__validate_pricing_rules(default_conf, caplog) -> None: validate_config_consistency(conf) +def test__validate_consumers(default_conf, caplog) -> None: + conf = deepcopy(default_conf) + conf.update({ + "external_message_consumer": { + "enabled": True, + "producers": [] + } + }) + with pytest.raises(OperationalException, + match="You must specify at least 1 Producer to connect to."): + validate_config_consistency(conf) + + conf = deepcopy(default_conf) + conf.update({ + "external_message_consumer": { + "enabled": True, + "producers": [ + { + "name": "default", + "host": "127.0.0.1", + "port": 8081, + "ws_token": "secret_ws_t0ken." + }, { + "name": "default", + "host": "127.0.0.1", + "port": 8080, + "ws_token": "secret_ws_t0ken." + } + ]} + }) + with pytest.raises(OperationalException, + match="Producer names must be unique. Duplicate: default"): + validate_config_consistency(conf) + + conf = deepcopy(default_conf) + conf.update({ + "process_only_new_candles": True, + "external_message_consumer": { + "enabled": True, + "producers": [ + { + "name": "default", + "host": "127.0.0.1", + "port": 8081, + "ws_token": "secret_ws_t0ken." + } + ]} + }) + validate_config_consistency(conf) + assert log_has_re("To receive best performance with external data.*", caplog) + + def test_load_config_test_comments() -> None: """ Load config with comments