adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into backtest_live_models

Browse Source
This commit is contained in:
Wagner Costa Santos 2022-09-24 12:30:55 -03:00
commit d9c16d4888
13 changed files with 120 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docker/Dockerfile.freqai
View File

@ -6,4 +6,3 @@ FROM ${sourceimage}:${sourcetag}
COPY requirements-freqai.txt /freqtrade/
RUN pip install -r requirements-freqai.txt --user --no-cache-dir

19
freqtrade/configuration/config_validation.py
View File

@ -1,4 +1,5 @@
import logging
from collections import Counter
from copy import deepcopy
from typing import Any, Dict
@ -85,6 +86,7 @@ def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False)
_validate_unlimited_amount(conf)
_validate_ask_orderbook(conf)
_validate_freqai_hyperopt(conf)
_validate_consumers(conf)
validate_migrated_strategy_settings(conf)
# validate configuration before returning
@ -332,6 +334,23 @@ def _validate_freqai_hyperopt(conf: Dict[str, Any]) -> None:
'Using analyze-per-epoch parameter is not supported with a FreqAI strategy.')
def _validate_consumers(conf: Dict[str, Any]) -> None:
emc_conf = conf.get('external_message_consumer', {})
if emc_conf.get('enabled', False):
if len(emc_conf.get('producers', [])) < 1:
raise OperationalException("You must specify at least 1 Producer to connect to.")
producer_names = [p['name'] for p in emc_conf.get('producers', [])]
duplicates = [item for item, count in Counter(producer_names).items() if count > 1]
if duplicates:
raise OperationalException(
f"Producer names must be unique. Duplicate: {', '.join(duplicates)}")
if conf.get('process_only_new_candles', True):
# Warning here or require it?
logger.warning("To receive best performance with external data, "
"please set `process_only_new_candles` to False")
def _strategy_settings(conf: Dict[str, Any]) -> None:
process_deprecated_setting(conf, None, 'use_sell_signal', None, 'use_exit_signal')

10
freqtrade/data/dataprovider.py
View File

@ -208,8 +208,10 @@ class DataProvider:
if saved_pair not in self.__cached_pairs_backtesting:
timerange = TimeRange.parse_timerange(None if self._config.get(
'timerange') is None else str(self._config.get('timerange')))
# Move informative start time respecting startup_candle_count
startup_candles = self.get_required_startup(str(timeframe))
# It is not necessary to add the training candles, as they
# were already added at the beginning of the backtest.
startup_candles = self.get_required_startup(str(timeframe), False)
tf_seconds = timeframe_to_seconds(str(timeframe))
timerange.subtract_start(tf_seconds * startup_candles)
self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
@ -223,7 +225,7 @@ class DataProvider:
)
return self.__cached_pairs_backtesting[saved_pair].copy()
def get_required_startup(self, timeframe: str) -> int:
def get_required_startup(self, timeframe: str, add_train_candles: bool = True) -> int:
freqai_config = self._config.get('freqai', {})
if not freqai_config.get('enabled', False):
return self._config.get('startup_candle_count', 0)
@ -233,6 +235,8 @@ class DataProvider:
# make sure the startupcandles is at least the set maximum indicator periods
self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods))
tf_seconds = timeframe_to_seconds(timeframe)
train_candles = 0
if add_train_candles:
train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds
total_candles = int(self._config['startup_candle_count'] + train_candles)
logger.info(f'Increasing startup_candle_count for freqai to {total_candles}')

4
freqtrade/exchange/exchange.py
View File

@ -2891,7 +2891,7 @@ def amount_to_contracts(amount: float, contract_size: Optional[float]) -> float:
:return: num-contracts
"""
if contract_size and contract_size != 1:
return amount / contract_size
return float(FtPrecise(amount) / FtPrecise(contract_size))
else:
return amount
@ -2905,7 +2905,7 @@ def contracts_to_amount(num_contracts: float, contract_size: Optional[float]) ->
"""
if contract_size and contract_size != 1:
return num_contracts * contract_size
return float(FtPrecise(num_contracts) * FtPrecise(contract_size))
else:
return num_contracts

23
freqtrade/freqai/data_kitchen.py
View File

@ -466,27 +466,6 @@ class FreqaiDataKitchen:
return df
def remove_training_from_backtesting(
self
) -> DataFrame:
"""
Function which takes the backtesting time range and
remove training data from dataframe, keeping only the
startup_candle_count candles
"""
startup_candle_count = self.config.get('startup_candle_count', 0)
tf = self.config['timeframe']
tr = self.config["timerange"]
backtesting_timerange = TimeRange.parse_timerange(tr)
if startup_candle_count > 0 and backtesting_timerange:
backtesting_timerange.subtract_start(timeframe_to_seconds(tf) * startup_candle_count)
start = datetime.fromtimestamp(backtesting_timerange.startts, tz=timezone.utc)
df = self.return_dataframe
df = df.loc[df["date"] >= start, :]
return df
def principal_component_analysis(self) -> None:
"""
Performs Principal Component Analysis on the data for dimensionality reduction
@ -994,8 +973,6 @@ class FreqaiDataKitchen:
to_keep = [col for col in dataframe.columns if not col.startswith("&")]
self.return_dataframe = pd.concat([dataframe[to_keep], self.full_df], axis=1)
self.return_dataframe = self.remove_training_from_backtesting()
self.full_df = DataFrame()
return

9
freqtrade/optimize/backtesting.py
View File

@ -139,9 +139,14 @@ class Backtesting:
# Get maximum required startup period
self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
if self.config.get('freqai', {}).get('enabled', False):
# For FreqAI, increase the required_startup to includes the training data
self.required_startup = self.dataprovider.get_required_startup(self.timeframe)
# Add maximum startup candle count to configuration for informative pairs support
self.config['startup_candle_count'] = self.required_startup
self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
self.trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT)
# strategies which define "can_short=True" will fail to load in Spot mode.
@ -217,7 +222,7 @@ class Backtesting:
pairs=self.pairlists.whitelist,
timeframe=self.timeframe,
timerange=self.timerange,
startup_candles=self.dataprovider.get_required_startup(self.timeframe),
startup_candles=self.config['startup_candle_count'],
fail_without_data=True,
data_format=self.config.get('dataformat_ohlcv', 'json'),
candle_type=self.config.get('candle_type_def', CandleType.SPOT)

35
freqtrade/rpc/external_message_consumer.py
View File

@ -8,14 +8,13 @@ import asyncio
import logging
import socket
from threading import Thread
from typing import TYPE_CHECKING, Any, Callable, Dict, List
from typing import TYPE_CHECKING, Any, Callable, Dict, List, TypedDict
import websockets
from pydantic import ValidationError
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import RPCMessageType
from freqtrade.exceptions import OperationalException
from freqtrade.misc import remove_entry_exit_signals
from freqtrade.rpc.api_server.ws import WebSocketChannel
from freqtrade.rpc.api_server.ws_schemas import (WSAnalyzedDFMessage, WSAnalyzedDFRequest,
@ -26,7 +25,13 @@ from freqtrade.rpc.api_server.ws_schemas import (WSAnalyzedDFMessage, WSAnalyzed
if TYPE_CHECKING:
import websockets.connect
import websockets.exceptions
class Producer(TypedDict):
name: str
host: str
port: int
ws_token: str
logger = logging.getLogger(__name__)
@ -55,7 +60,7 @@ class ExternalMessageConsumer:
self._emc_config = self._config.get('external_message_consumer', {})
self.enabled = self._emc_config.get('enabled', False)
self.producers = self._emc_config.get('producers', [])
self.producers: List[Producer] = self._emc_config.get('producers', [])
self.wait_timeout = self._emc_config.get('wait_timeout', 300) # in seconds
self.ping_timeout = self._emc_config.get('ping_timeout', 10) # in seconds
@ -68,8 +73,6 @@ class ExternalMessageConsumer:
# as the websockets client expects bytes.
self.message_size_limit = (self._emc_config.get('message_size_limit', 8) << 20)
self.validate_config()
# Setting these explicitly as they probably shouldn't be changed by a user
# Unless we somehow integrate this with the strategy to allow creating
# callbacks for the messages
@ -90,18 +93,6 @@ class ExternalMessageConsumer:
self.start()
def validate_config(self):
"""
Make sure values are what they are supposed to be
"""
if self.enabled and len(self.producers) < 1:
raise OperationalException("You must specify at least 1 Producer to connect to.")
if self.enabled and self._config.get('process_only_new_candles', True):
# Warning here or require it?
logger.warning("To receive best performance with external data, "
"please set `process_only_new_candles` to False")
def start(self):
"""
Start the main internal loop in another thread to run coroutines
@ -162,7 +153,7 @@ class ExternalMessageConsumer:
# Stop the loop once we are done
self._loop.stop()
async def _handle_producer_connection(self, producer: Dict[str, Any], lock: asyncio.Lock):
async def _handle_producer_connection(self, producer: Producer, lock: asyncio.Lock):
"""
Main connection loop for the consumer
@ -175,7 +166,7 @@ class ExternalMessageConsumer:
# Exit silently
pass
async def _create_connection(self, producer: Dict[str, Any], lock: asyncio.Lock):
async def _create_connection(self, producer: Producer, lock: asyncio.Lock):
"""
Actually creates and handles the websocket connection, pinging on timeout
and handling connection errors.
@ -236,7 +227,7 @@ class ExternalMessageConsumer:
async def _receive_messages(
self,
channel: WebSocketChannel,
producer: Dict[str, Any],
producer: Producer,
lock: asyncio.Lock
):
"""
@ -277,7 +268,7 @@ class ExternalMessageConsumer:
break
def handle_producer_message(self, producer: Dict[str, Any], message: Dict[str, Any]):
def handle_producer_message(self, producer: Producer, message: Dict[str, Any]):
"""
Handles external messages from a Producer
"""

1
setup.cfg
View File

@ -49,4 +49,3 @@ exclude =
__pycache__,
.eggs,
user_data,

16
tests/exchange/test_exchange.py
View File

@ -20,6 +20,7 @@ from freqtrade.exchange import (Binance, Bittrex, Exchange, Kraken, amount_to_pr
timeframe_to_prev_date, timeframe_to_seconds)
from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT,
calculate_backoff, remove_credentials)
from freqtrade.exchange.exchange import amount_to_contract_precision
from freqtrade.resolvers.exchange_resolver import ExchangeResolver
from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re, num_log_has_re
@ -4470,6 +4471,7 @@ def test__amount_to_contracts(
('ADA/USDT:USDT', 10.4445555, 10.4, 10.444),
('LTC/ETH', 30, 30, 30),
('LTC/USD', 30, 30, 30),
('ADA/USDT:USDT', 1.17, 1.1, 1.17),
# contract size of 10
('ETH/USDT:USDT', 10.111, 10.1, 10),
('ETH/USDT:USDT', 10.188, 10.1, 10),
@ -4497,6 +4499,20 @@ def test_amount_to_contract_precision(
assert result_size == expected_fut
@pytest.mark.parametrize('amount,precision,precision_mode,contract_size,expected', [
(1.17, 1.0, 4, 0.01, 1.17), # Tick size
(1.17, 1.0, 2, 0.01, 1.17), #
(1.16, 1.0, 4, 0.01, 1.16), #
(1.16, 1.0, 2, 0.01, 1.16), #
(1.13, 1.0, 2, 0.01, 1.13), #
(10.988, 1.0, 2, 10, 10),
(10.988, 1.0, 4, 10, 10),
])
def test_amount_to_contract_precision2(amount, precision, precision_mode, contract_size, expected):
res = amount_to_contract_precision(amount, precision, precision_mode, contract_size)
assert pytest.approx(res) == expected
@pytest.mark.parametrize('exchange_name,open_rate,is_short,trading_mode,margin_mode', [
# Bittrex
('bittrex', 2.0, False, 'spot', None),

14
tests/rpc/test_rpc_emc.py
View File

@ -11,7 +11,6 @@ import pytest
import websockets
from freqtrade.data.dataprovider import DataProvider
from freqtrade.exceptions import OperationalException
from freqtrade.rpc.external_message_consumer import ExternalMessageConsumer
from tests.conftest import log_has, log_has_re, log_has_when
@ -73,23 +72,12 @@ def test_emc_shutdown(patched_emc, caplog):
assert not log_has("Stopping ExternalMessageConsumer", caplog)
def test_emc_init(patched_emc, default_conf):
def test_emc_init(patched_emc):
# Test the settings were set correctly
assert patched_emc.initial_candle_limit <= 1500
assert patched_emc.wait_timeout > 0
assert patched_emc.sleep_time > 0
default_conf.update({
"external_message_consumer": {
"enabled": True,
"producers": []
}
})
dataprovider = DataProvider(default_conf, None, None, None)
with pytest.raises(OperationalException,
match="You must specify at least 1 Producer to connect to."):
ExternalMessageConsumer(default_conf, dataprovider)
# Parametrize this?
def test_emc_handle_producer_message(patched_emc, caplog, ohlcv_history):

52
tests/test_configuration.py
View File

@ -1089,6 +1089,58 @@ def test__validate_pricing_rules(default_conf, caplog) -> None:
validate_config_consistency(conf)
def test__validate_consumers(default_conf, caplog) -> None:
conf = deepcopy(default_conf)
conf.update({
"external_message_consumer": {
"enabled": True,
"producers": []
}
})
with pytest.raises(OperationalException,
match="You must specify at least 1 Producer to connect to."):
validate_config_consistency(conf)
conf = deepcopy(default_conf)
conf.update({
"external_message_consumer": {
"enabled": True,
"producers": [
{
"name": "default",
"host": "127.0.0.1",
"port": 8081,
"ws_token": "secret_ws_t0ken."
}, {
"name": "default",
"host": "127.0.0.1",
"port": 8080,
"ws_token": "secret_ws_t0ken."
}
]}
})
with pytest.raises(OperationalException,
match="Producer names must be unique. Duplicate: default"):
validate_config_consistency(conf)
conf = deepcopy(default_conf)
conf.update({
"process_only_new_candles": True,
"external_message_consumer": {
"enabled": True,
"producers": [
{
"name": "default",
"host": "127.0.0.1",
"port": 8081,
"ws_token": "secret_ws_t0ken."
}
]}
})
validate_config_consistency(conf)
assert log_has_re("To receive best performance with external data.*", caplog)
def test_load_config_test_comments() -> None:
"""
Load config with comments