Merge branch 'develop' into backtest_live_models

config_examples/config_freqai.example.json

@@ -94,4 +94,4 @@
     "internals": {
         "process_throttle_secs": 5
     }
-}
+}

docker/Dockerfile.freqai

@@ -6,4 +6,3 @@ FROM ${sourceimage}:${sourcetag}
 COPY requirements-freqai.txt /freqtrade/
 
 RUN pip install -r requirements-freqai.txt --user --no-cache-dir
-

freqtrade/configuration/config_validation.py

@@ -1,4 +1,5 @@
 import logging
+from collections import Counter
 from copy import deepcopy
 from typing import Any, Dict
 
@@ -85,6 +86,7 @@ def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False)
     _validate_unlimited_amount(conf)
     _validate_ask_orderbook(conf)
     _validate_freqai_hyperopt(conf)
+    _validate_consumers(conf)
     validate_migrated_strategy_settings(conf)
 
     # validate configuration before returning
@@ -332,6 +334,23 @@ def _validate_freqai_hyperopt(conf: Dict[str, Any]) -> None:
             'Using analyze-per-epoch parameter is not supported with a FreqAI strategy.')
 
 
+def _validate_consumers(conf: Dict[str, Any]) -> None:
+    emc_conf = conf.get('external_message_consumer', {})
+    if emc_conf.get('enabled', False):
+        if len(emc_conf.get('producers', [])) < 1:
+            raise OperationalException("You must specify at least 1 Producer to connect to.")
+
+        producer_names = [p['name'] for p in emc_conf.get('producers', [])]
+        duplicates = [item for item, count in Counter(producer_names).items() if count > 1]
+        if duplicates:
+            raise OperationalException(
+                f"Producer names must be unique. Duplicate: {', '.join(duplicates)}")
+        if conf.get('process_only_new_candles', True):
+            # Warning here or require it?
+            logger.warning("To receive best performance with external data, "
+                           "please set `process_only_new_candles` to False")
+
+
 def _strategy_settings(conf: Dict[str, Any]) -> None:
 
     process_deprecated_setting(conf, None, 'use_sell_signal', None, 'use_exit_signal')

freqtrade/data/dataprovider.py

@@ -208,8 +208,10 @@ class DataProvider:
         if saved_pair not in self.__cached_pairs_backtesting:
             timerange = TimeRange.parse_timerange(None if self._config.get(
                 'timerange') is None else str(self._config.get('timerange')))
-            # Move informative start time respecting startup_candle_count
+
-            startup_candles = self.get_required_startup(str(timeframe))
+            # It is not necessary to add the training candles, as they
+            # were already added at the beginning of the backtest.
+            startup_candles = self.get_required_startup(str(timeframe), False)
             tf_seconds = timeframe_to_seconds(str(timeframe))
             timerange.subtract_start(tf_seconds * startup_candles)
             self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
@@ -223,7 +225,7 @@ class DataProvider:
             )
         return self.__cached_pairs_backtesting[saved_pair].copy()
 
-    def get_required_startup(self, timeframe: str) -> int:
+    def get_required_startup(self, timeframe: str, add_train_candles: bool = True) -> int:
         freqai_config = self._config.get('freqai', {})
         if not freqai_config.get('enabled', False):
             return self._config.get('startup_candle_count', 0)
@@ -233,7 +235,9 @@ class DataProvider:
             # make sure the startupcandles is at least the set maximum indicator periods
             self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods))
             tf_seconds = timeframe_to_seconds(timeframe)
-            train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds
+            train_candles = 0
+            if add_train_candles:
+                train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds
             total_candles = int(self._config['startup_candle_count'] + train_candles)
             logger.info(f'Increasing startup_candle_count for freqai to {total_candles}')
             return total_candles

freqtrade/exchange/binance_leverage_tiers.json

@@ -19209,4 +19209,4 @@
       }
     }
   ]
-}
+}

freqtrade/exchange/exchange.py

@@ -2891,7 +2891,7 @@ def amount_to_contracts(amount: float, contract_size: Optional[float]) -> float:
     :return: num-contracts
     """
     if contract_size and contract_size != 1:
-        return amount / contract_size
+        return float(FtPrecise(amount) / FtPrecise(contract_size))
     else:
         return amount
 
@@ -2905,7 +2905,7 @@ def contracts_to_amount(num_contracts: float, contract_size: Optional[float]) ->
     """
 
     if contract_size and contract_size != 1:
-        return num_contracts * contract_size
+        return float(FtPrecise(num_contracts) * FtPrecise(contract_size))
     else:
         return num_contracts
 

freqtrade/freqai/data_kitchen.py

@@ -466,27 +466,6 @@ class FreqaiDataKitchen:
 
         return df
 
-    def remove_training_from_backtesting(
-        self
-    ) -> DataFrame:
-        """
-        Function which takes the backtesting time range and
-        remove training data from dataframe, keeping only the
-        startup_candle_count candles
-        """
-        startup_candle_count = self.config.get('startup_candle_count', 0)
-        tf = self.config['timeframe']
-        tr = self.config["timerange"]
-
-        backtesting_timerange = TimeRange.parse_timerange(tr)
-        if startup_candle_count > 0 and backtesting_timerange:
-            backtesting_timerange.subtract_start(timeframe_to_seconds(tf) * startup_candle_count)
-
-        start = datetime.fromtimestamp(backtesting_timerange.startts, tz=timezone.utc)
-        df = self.return_dataframe
-        df = df.loc[df["date"] >= start, :]
-        return df
-
     def principal_component_analysis(self) -> None:
         """
         Performs Principal Component Analysis on the data for dimensionality reduction
@@ -994,8 +973,6 @@ class FreqaiDataKitchen:
 
         to_keep = [col for col in dataframe.columns if not col.startswith("&")]
         self.return_dataframe = pd.concat([dataframe[to_keep], self.full_df], axis=1)
-
-        self.return_dataframe = self.remove_training_from_backtesting()
         self.full_df = DataFrame()
 
         return

freqtrade/optimize/backtesting.py

@@ -139,9 +139,14 @@ class Backtesting:
 
         # Get maximum required startup period
         self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
+        self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
+
+        if self.config.get('freqai', {}).get('enabled', False):
+            # For FreqAI, increase the required_startup to includes the training data
+            self.required_startup = self.dataprovider.get_required_startup(self.timeframe)
+
         # Add maximum startup candle count to configuration for informative pairs support
         self.config['startup_candle_count'] = self.required_startup
-        self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
 
         self.trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT)
         # strategies which define "can_short=True" will fail to load in Spot mode.
@@ -217,7 +222,7 @@ class Backtesting:
             pairs=self.pairlists.whitelist,
             timeframe=self.timeframe,
             timerange=self.timerange,
-            startup_candles=self.dataprovider.get_required_startup(self.timeframe),
+            startup_candles=self.config['startup_candle_count'],
             fail_without_data=True,
             data_format=self.config.get('dataformat_ohlcv', 'json'),
             candle_type=self.config.get('candle_type_def', CandleType.SPOT)

freqtrade/rpc/external_message_consumer.py

@@ -8,14 +8,13 @@ import asyncio
 import logging
 import socket
 from threading import Thread
-from typing import TYPE_CHECKING, Any, Callable, Dict, List
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, TypedDict
 
 import websockets
 from pydantic import ValidationError
 
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import RPCMessageType
-from freqtrade.exceptions import OperationalException
 from freqtrade.misc import remove_entry_exit_signals
 from freqtrade.rpc.api_server.ws import WebSocketChannel
 from freqtrade.rpc.api_server.ws_schemas import (WSAnalyzedDFMessage, WSAnalyzedDFRequest,
@@ -26,7 +25,13 @@ from freqtrade.rpc.api_server.ws_schemas import (WSAnalyzedDFMessage, WSAnalyzed
 
 if TYPE_CHECKING:
     import websockets.connect
-    import websockets.exceptions
+
+
+class Producer(TypedDict):
+    name: str
+    host: str
+    port: int
+    ws_token: str
 
 
 logger = logging.getLogger(__name__)
@@ -55,7 +60,7 @@ class ExternalMessageConsumer:
         self._emc_config = self._config.get('external_message_consumer', {})
 
         self.enabled = self._emc_config.get('enabled', False)
-        self.producers = self._emc_config.get('producers', [])
+        self.producers: List[Producer] = self._emc_config.get('producers', [])
 
         self.wait_timeout = self._emc_config.get('wait_timeout', 300)  # in seconds
         self.ping_timeout = self._emc_config.get('ping_timeout', 10)  # in seconds
@@ -68,8 +73,6 @@ class ExternalMessageConsumer:
         # as the websockets client expects bytes.
         self.message_size_limit = (self._emc_config.get('message_size_limit', 8) << 20)
 
-        self.validate_config()
-
         # Setting these explicitly as they probably shouldn't be changed by a user
         # Unless we somehow integrate this with the strategy to allow creating
         # callbacks for the messages
@@ -90,18 +93,6 @@ class ExternalMessageConsumer:
 
         self.start()
 
-    def validate_config(self):
-        """
-        Make sure values are what they are supposed to be
-        """
-        if self.enabled and len(self.producers) < 1:
-            raise OperationalException("You must specify at least 1 Producer to connect to.")
-
-        if self.enabled and self._config.get('process_only_new_candles', True):
-            # Warning here or require it?
-            logger.warning("To receive best performance with external data, "
-                           "please set `process_only_new_candles` to False")
-
     def start(self):
         """
         Start the main internal loop in another thread to run coroutines
@@ -162,7 +153,7 @@ class ExternalMessageConsumer:
             # Stop the loop once we are done
             self._loop.stop()
 
-    async def _handle_producer_connection(self, producer: Dict[str, Any], lock: asyncio.Lock):
+    async def _handle_producer_connection(self, producer: Producer, lock: asyncio.Lock):
         """
         Main connection loop for the consumer
 
@@ -175,7 +166,7 @@ class ExternalMessageConsumer:
             # Exit silently
             pass
 
-    async def _create_connection(self, producer: Dict[str, Any], lock: asyncio.Lock):
+    async def _create_connection(self, producer: Producer, lock: asyncio.Lock):
         """
         Actually creates and handles the websocket connection, pinging on timeout
         and handling connection errors.
@@ -236,7 +227,7 @@ class ExternalMessageConsumer:
     async def _receive_messages(
         self,
         channel: WebSocketChannel,
-        producer: Dict[str, Any],
+        producer: Producer,
         lock: asyncio.Lock
     ):
         """
@@ -277,7 +268,7 @@ class ExternalMessageConsumer:
 
                     break
 
-    def handle_producer_message(self, producer: Dict[str, Any], message: Dict[str, Any]):
+    def handle_producer_message(self, producer: Producer, message: Dict[str, Any]):
         """
         Handles external messages from a Producer
         """

setup.cfg

@@ -49,4 +49,3 @@ exclude =
     __pycache__,
     .eggs,
     user_data,
-

tests/exchange/test_exchange.py

@@ -20,6 +20,7 @@ from freqtrade.exchange import (Binance, Bittrex, Exchange, Kraken, amount_to_pr
                                 timeframe_to_prev_date, timeframe_to_seconds)
 from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT,
                                        calculate_backoff, remove_credentials)
+from freqtrade.exchange.exchange import amount_to_contract_precision
 from freqtrade.resolvers.exchange_resolver import ExchangeResolver
 from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re, num_log_has_re
 
@@ -4470,6 +4471,7 @@ def test__amount_to_contracts(
     ('ADA/USDT:USDT', 10.4445555, 10.4, 10.444),
     ('LTC/ETH', 30, 30, 30),
     ('LTC/USD', 30, 30, 30),
+    ('ADA/USDT:USDT', 1.17, 1.1, 1.17),
     # contract size of 10
     ('ETH/USDT:USDT', 10.111, 10.1, 10),
     ('ETH/USDT:USDT', 10.188, 10.1, 10),
@@ -4497,6 +4499,20 @@ def test_amount_to_contract_precision(
     assert result_size == expected_fut
 
 
+@pytest.mark.parametrize('amount,precision,precision_mode,contract_size,expected', [
+    (1.17, 1.0, 4, 0.01, 1.17),  # Tick size
+    (1.17, 1.0, 2, 0.01, 1.17),  #
+    (1.16, 1.0, 4, 0.01, 1.16),  #
+    (1.16, 1.0, 2, 0.01, 1.16),  #
+    (1.13, 1.0, 2, 0.01, 1.13),  #
+    (10.988, 1.0, 2, 10, 10),
+    (10.988, 1.0, 4, 10, 10),
+])
+def test_amount_to_contract_precision2(amount, precision, precision_mode, contract_size, expected):
+    res = amount_to_contract_precision(amount, precision, precision_mode, contract_size)
+    assert pytest.approx(res) == expected
+
+
 @pytest.mark.parametrize('exchange_name,open_rate,is_short,trading_mode,margin_mode', [
     # Bittrex
     ('bittrex', 2.0, False, 'spot', None),

tests/rpc/test_rpc_emc.py

@@ -11,7 +11,6 @@ import pytest
 import websockets
 
 from freqtrade.data.dataprovider import DataProvider
-from freqtrade.exceptions import OperationalException
 from freqtrade.rpc.external_message_consumer import ExternalMessageConsumer
 from tests.conftest import log_has, log_has_re, log_has_when
 
@@ -73,23 +72,12 @@ def test_emc_shutdown(patched_emc, caplog):
     assert not log_has("Stopping ExternalMessageConsumer", caplog)
 
 
-def test_emc_init(patched_emc, default_conf):
+def test_emc_init(patched_emc):
     # Test the settings were set correctly
     assert patched_emc.initial_candle_limit <= 1500
     assert patched_emc.wait_timeout > 0
     assert patched_emc.sleep_time > 0
 
-    default_conf.update({
-        "external_message_consumer": {
-            "enabled": True,
-            "producers": []
-        }
-    })
-    dataprovider = DataProvider(default_conf, None, None, None)
-    with pytest.raises(OperationalException,
-                       match="You must specify at least 1 Producer to connect to."):
-        ExternalMessageConsumer(default_conf, dataprovider)
-
 
 # Parametrize this?
 def test_emc_handle_producer_message(patched_emc, caplog, ohlcv_history):

tests/test_configuration.py

@@ -1089,6 +1089,58 @@ def test__validate_pricing_rules(default_conf, caplog) -> None:
         validate_config_consistency(conf)
 
 
+def test__validate_consumers(default_conf, caplog) -> None:
+    conf = deepcopy(default_conf)
+    conf.update({
+            "external_message_consumer": {
+                "enabled": True,
+                "producers": []
+                }
+            })
+    with pytest.raises(OperationalException,
+                       match="You must specify at least 1 Producer to connect to."):
+        validate_config_consistency(conf)
+
+    conf = deepcopy(default_conf)
+    conf.update({
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": "127.0.0.1",
+                    "port": 8081,
+                    "ws_token": "secret_ws_t0ken."
+                }, {
+                    "name": "default",
+                    "host": "127.0.0.1",
+                    "port": 8080,
+                    "ws_token": "secret_ws_t0ken."
+                }
+            ]}
+        })
+    with pytest.raises(OperationalException,
+                       match="Producer names must be unique. Duplicate: default"):
+        validate_config_consistency(conf)
+
+    conf = deepcopy(default_conf)
+    conf.update({
+        "process_only_new_candles": True,
+        "external_message_consumer": {
+            "enabled": True,
+            "producers": [
+                {
+                    "name": "default",
+                    "host": "127.0.0.1",
+                    "port": 8081,
+                    "ws_token": "secret_ws_t0ken."
+                }
+            ]}
+        })
+    validate_config_consistency(conf)
+    assert log_has_re("To receive best performance with external data.*", caplog)
+
+
 def test_load_config_test_comments() -> None:
     """
     Load config with comments