From 8a3272e7c54ec31572d039124a8fe052ec83a3d9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 17 Oct 2018 19:47:19 +0200 Subject: [PATCH 1/7] don't copy tickerdata_to_dataframe into backtesting it's used only once, so this does not make sense and hides the origin of the function --- freqtrade/optimize/backtesting.py | 3 +-- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 16 ++++++++-------- freqtrade/tests/optimize/test_hyperopt.py | 6 +++--- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cd822023f..df2f6834a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -88,7 +88,6 @@ class Backtesting(object): """ self.strategy = strategy self.ticker_interval = self.config.get('ticker_interval') - self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe self.advise_buy = strategy.advise_buy self.advise_sell = strategy.advise_sell @@ -371,7 +370,7 @@ class Backtesting(object): self._set_strategy(strat) # need to reprocess data every time to populate signals - preprocessed = self.tickerdata_to_dataframe(data) + preprocessed = self.strategy.tickerdata_to_dataframe(data) # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 4a239ab28..b2d05d603 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -352,7 +352,7 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.strategy.advise_indicators = Hyperopt.populate_indicators # type: ignore - dump(self.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) + dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index a17867b3a..36b2fcdd3 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -89,7 +89,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None: backtesting = Backtesting(config) data = load_data_test(contour) - processed = backtesting.tickerdata_to_dataframe(data) + processed = backtesting.strategy.tickerdata_to_dataframe(data) assert isinstance(processed, dict) results = backtesting.backtest( { @@ -125,7 +125,7 @@ def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): backtesting = Backtesting(conf) return { 'stake_amount': conf['stake_amount'], - 'processed': backtesting.tickerdata_to_dataframe(data), + 'processed': backtesting.strategy.tickerdata_to_dataframe(data), 'max_open_trades': 10, 'position_stacking': False, 'record': record @@ -313,7 +313,7 @@ def test_backtesting_init(mocker, default_conf) -> None: backtesting = Backtesting(default_conf) assert backtesting.config == default_conf assert backtesting.ticker_interval == '5m' - assert callable(backtesting.tickerdata_to_dataframe) + assert callable(backtesting.strategy.tickerdata_to_dataframe) assert callable(backtesting.advise_buy) assert callable(backtesting.advise_sell) get_fee.assert_called() @@ -327,7 +327,7 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: tickerlist = {'UNITTEST/BTC': tick} backtesting = Backtesting(default_conf) - data = backtesting.tickerdata_to_dataframe(tickerlist) + data = backtesting.strategy.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 99 # Load strategy to compare the result between Backtesting function and strategy are the same @@ -340,7 +340,7 @@ def test_get_timeframe(default_conf, mocker) -> None: patch_exchange(mocker) backtesting = Backtesting(default_conf) - data = backtesting.tickerdata_to_dataframe( + data = backtesting.strategy.tickerdata_to_dataframe( optimize.load_data( None, ticker_interval='1m', @@ -520,7 +520,7 @@ def test_backtest(default_conf, fee, mocker) -> None: pair = 'UNITTEST/BTC' data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) data = trim_dictlist(data, -200) - data_processed = backtesting.tickerdata_to_dataframe(data) + data_processed = backtesting.strategy.tickerdata_to_dataframe(data) results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], @@ -571,7 +571,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: results = backtesting.backtest( { 'stake_amount': default_conf['stake_amount'], - 'processed': backtesting.tickerdata_to_dataframe(data), + 'processed': backtesting.strategy.tickerdata_to_dataframe(data), 'max_open_trades': 1, 'position_stacking': False } @@ -585,7 +585,7 @@ def test_processed(default_conf, mocker) -> None: backtesting = Backtesting(default_conf) dict_of_tickerrows = load_data_test('raise') - dataframes = backtesting.tickerdata_to_dataframe(dict_of_tickerrows) + dataframes = backtesting.strategy.tickerdata_to_dataframe(dict_of_tickerrows) dataframe = dataframes['UNITTEST/BTC'] cols = dataframe.columns # assert the dataframe got some of the indicator columns diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 2035e23df..c93f2d316 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -194,7 +194,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: default_conf.update({'spaces': 'all'}) hyperopt = Hyperopt(default_conf) - hyperopt.tickerdata_to_dataframe = MagicMock() + hyperopt.strategy.tickerdata_to_dataframe = MagicMock() hyperopt.start() parallel.assert_called_once() @@ -242,7 +242,7 @@ def test_has_space(hyperopt): def test_populate_indicators(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} - dataframes = hyperopt.tickerdata_to_dataframe(tickerlist) + dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) # Check if some indicators are generated. We will not test all of them @@ -254,7 +254,7 @@ def test_populate_indicators(hyperopt) -> None: def test_buy_strategy_generator(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} - dataframes = hyperopt.tickerdata_to_dataframe(tickerlist) + dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) populate_buy_trend = hyperopt.buy_strategy_generator( From d7459bbbf3c60695c22201b2273c3a77b4385dea Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 17 Oct 2018 19:59:33 +0200 Subject: [PATCH 2/7] refactor get_timeframe out of backtesting class --- freqtrade/optimize/__init__.py | 17 ++++++++++++++ freqtrade/optimize/backtesting.py | 20 ++-------------- freqtrade/tests/optimize/test_backtesting.py | 24 ++++---------------- freqtrade/tests/optimize/test_optimize.py | 19 +++++++++++++++- 4 files changed, 41 insertions(+), 39 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 74c842427..5367f7663 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -11,7 +11,10 @@ except ImportError: import logging import os from typing import Optional, List, Dict, Tuple, Any +import operator + import arrow +from pandas import DataFrame from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange @@ -59,6 +62,20 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: return tickerlist[start_index:stop_index] +def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: + """ + Get the maximum timeframe for the given backtest data + :param data: dictionary with preprocessed backtesting data + :return: tuple containing min_date, max_date + """ + timeframe = [ + (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) + for frame in data.values() + ] + return min(timeframe, key=operator.itemgetter(0))[0], \ + max(timeframe, key=operator.itemgetter(1))[1] + + def load_tickerdata_file( datadir: str, pair: str, ticker_interval: str, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index df2f6834a..695a52052 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -4,14 +4,12 @@ This module contains the backtesting logic """ import logging -import operator from argparse import Namespace from copy import deepcopy from datetime import datetime, timedelta from pathlib import Path -from typing import Any, Dict, List, NamedTuple, Optional, Tuple +from typing import Any, Dict, List, NamedTuple, Optional -import arrow from pandas import DataFrame from tabulate import tabulate @@ -91,20 +89,6 @@ class Backtesting(object): self.advise_buy = strategy.advise_buy self.advise_sell = strategy.advise_sell - @staticmethod - def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: - """ - Get the maximum timeframe for the given backtest data - :param data: dictionary with preprocessed backtesting data - :return: tuple containing min_date, max_date - """ - timeframe = [ - (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) - for frame in data.values() - ] - return min(timeframe, key=operator.itemgetter(0))[0], \ - max(timeframe, key=operator.itemgetter(1))[1] - def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame, skip_nan: bool = False) -> str: """ @@ -373,7 +357,7 @@ class Backtesting(object): preprocessed = self.strategy.tickerdata_to_dataframe(data) # Print timeframe - min_date, max_date = self.get_timeframe(preprocessed) + min_date, max_date = optimize.get_timeframe(preprocessed) logger.info( 'Measuring data from %s up to %s (%s days)..', min_date.isoformat(), diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 36b2fcdd3..ff6a0666d 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -336,22 +336,6 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None: assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC']) -def test_get_timeframe(default_conf, mocker) -> None: - patch_exchange(mocker) - backtesting = Backtesting(default_conf) - - data = backtesting.strategy.tickerdata_to_dataframe( - optimize.load_data( - None, - ticker_interval='1m', - pairs=['UNITTEST/BTC'] - ) - ) - min_date, max_date = backtesting.get_timeframe(data) - assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' - assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' - - def test_generate_text_table(default_conf, mocker): patch_exchange(mocker) backtesting = Backtesting(default_conf) @@ -451,17 +435,17 @@ def test_generate_text_table_strategyn(default_conf, mocker): def test_backtesting_start(default_conf, mocker, caplog) -> None: - def get_timeframe(input1, input2): + def get_timeframe(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) + mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe) mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', backtest=MagicMock(), _generate_text_table=MagicMock(return_value='1'), - get_timeframe=get_timeframe, ) default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] @@ -486,17 +470,17 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: - def get_timeframe(input1, input2): + def get_timeframe(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) + mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe) mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', backtest=MagicMock(), _generate_text_table=MagicMock(return_value='1'), - get_timeframe=get_timeframe, ) default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 77fa3e3b1..061caf70b 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -15,7 +15,8 @@ from freqtrade.optimize.__init__ import (download_backtesting_testdata, load_cached_data_for_updating, load_tickerdata_file, make_testdata_path, trim_tickerlist) -from freqtrade.tests.conftest import get_patched_exchange, log_has +from freqtrade.strategy.default_strategy import DefaultStrategy +from freqtrade.tests.conftest import get_patched_exchange, log_has, patch_exchange # Change this if modifying UNITTEST/BTC testdatafile _BTC_UNITTEST_LENGTH = 13681 @@ -433,3 +434,19 @@ def test_file_dump_json() -> None: # Remove the file _clean_test_file(file) + + +def test_get_timeframe(default_conf, mocker) -> None: + patch_exchange(mocker) + strategy = DefaultStrategy(default_conf) + + data = strategy.tickerdata_to_dataframe( + optimize.load_data( + None, + ticker_interval='1m', + pairs=['UNITTEST/BTC'] + ) + ) + min_date, max_date = optimize.get_timeframe(data) + assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' + assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' From fb52d322966382e09c39367c2a10ffbd8efc0aaa Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 Oct 2018 19:42:54 +0200 Subject: [PATCH 3/7] Add validate_backtest_data function --- freqtrade/optimize/__init__.py | 19 ++++++++++ freqtrade/optimize/backtesting.py | 4 ++- freqtrade/tests/optimize/test_optimize.py | 44 ++++++++++++++++++++++- 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 5367f7663..d4cb6c067 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -10,6 +10,7 @@ except ImportError: _UJSON = False import logging import os +from datetime import datetime from typing import Optional, List, Dict, Tuple, Any import operator @@ -76,6 +77,24 @@ def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow] max(timeframe, key=operator.itemgetter(1))[1] +def validate_backtest_data(data: Dict[str, DataFrame], min_date: datetime, + max_date: datetime, ticker_interval_mins: int) -> None: + """ + Validates preprocessed backtesting data for missing values and shows warnings about it that. + + :param data: dictionary with preprocessed backtesting data + :param min_date: start-date of the data + :param max_date: end-date of the data + :param ticker_interval_mins: ticker interval in minutes + """ + # total difference in minutes / interval-minutes + expected_frames = int((max_date - min_date).total_seconds() // 60 // ticker_interval_mins) + for pair, df in data.items(): + if len(df) < expected_frames: + logger.warning('%s has missing frames: expected %s, got %s', + pair, expected_frames, len(df)) + + def load_tickerdata_file( datadir: str, pair: str, ticker_interval: str, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 695a52052..961cfb092 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -356,8 +356,10 @@ class Backtesting(object): # need to reprocess data every time to populate signals preprocessed = self.strategy.tickerdata_to_dataframe(data) - # Print timeframe min_date, max_date = optimize.get_timeframe(preprocessed) + # Validate dataframe for missing values + optimize.validate_backtest_data(preprocessed, min_date, max_date, + constants.TICKER_INTERVAL_MINUTES[self.ticker_interval]) logger.info( 'Measuring data from %s up to %s (%s days)..', min_date.isoformat(), diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 061caf70b..7b13b498a 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -7,7 +7,7 @@ from shutil import copyfile import arrow -from freqtrade import optimize +from freqtrade import optimize, constants from freqtrade.arguments import TimeRange from freqtrade.misc import file_dump_json from freqtrade.optimize.__init__ import (download_backtesting_testdata, @@ -450,3 +450,45 @@ def test_get_timeframe(default_conf, mocker) -> None: min_date, max_date = optimize.get_timeframe(data) assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' + + +def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: + patch_exchange(mocker) + strategy = DefaultStrategy(default_conf) + + data = strategy.tickerdata_to_dataframe( + optimize.load_data( + None, + ticker_interval='1m', + pairs=['UNITTEST/BTC'] + ) + ) + min_date, max_date = optimize.get_timeframe(data) + caplog.clear() + optimize.validate_backtest_data(data, min_date, max_date, + constants.TICKER_INTERVAL_MINUTES["1m"]) + assert len(caplog.record_tuples) == 1 + assert log_has('UNITTEST/BTC has missing frames: expected 14396, got 13680', + caplog.record_tuples) + + +def test_validate_backtest_data(default_conf, mocker, caplog) -> None: + patch_exchange(mocker) + strategy = DefaultStrategy(default_conf) + + timerange = TimeRange('index', 'index', 200, 250) + data = strategy.tickerdata_to_dataframe( + optimize.load_data( + None, + ticker_interval='5m', + pairs=['UNITTEST/BTC'], + timerange=timerange + ) + ) + + min_date, max_date = optimize.get_timeframe(data) + caplog.clear() + optimize.validate_backtest_data(data, min_date, max_date, + constants.TICKER_INTERVAL_MINUTES["5m"]) + assert len(caplog.record_tuples) == 0 + From 518dcf5209034e4c7fdf14f99257d406363fdcdb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 Oct 2018 19:43:04 +0200 Subject: [PATCH 4/7] Cleanup some tests 8m is not a valid ticker value not in constants.TICKER_INTERVAL_MINUTES map --- freqtrade/tests/optimize/test_backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index ff6a0666d..83fe52ed3 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -119,7 +119,7 @@ def _load_pair_as_ticks(pair, tickfreq): # FIX: fixturize this? def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): - data = optimize.load_data(None, ticker_interval='8m', pairs=[pair]) + data = optimize.load_data(None, ticker_interval='1m', pairs=[pair]) data = trim_dictlist(data, -201) patch_exchange(mocker) backtesting = Backtesting(conf) @@ -449,7 +449,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: ) default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - default_conf['ticker_interval'] = 1 + default_conf['ticker_interval'] = "1m" default_conf['live'] = False default_conf['datadir'] = None default_conf['export'] = None From bc356c4d6511034f691793f54f26c59fdf2a2dc9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 Oct 2018 19:48:54 +0200 Subject: [PATCH 5/7] Return true/false for validation function --- freqtrade/optimize/__init__.py | 5 ++++- freqtrade/tests/optimize/test_optimize.py | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index d4cb6c067..3376d0075 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -78,7 +78,7 @@ def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow] def validate_backtest_data(data: Dict[str, DataFrame], min_date: datetime, - max_date: datetime, ticker_interval_mins: int) -> None: + max_date: datetime, ticker_interval_mins: int) -> bool: """ Validates preprocessed backtesting data for missing values and shows warnings about it that. @@ -89,10 +89,13 @@ def validate_backtest_data(data: Dict[str, DataFrame], min_date: datetime, """ # total difference in minutes / interval-minutes expected_frames = int((max_date - min_date).total_seconds() // 60 // ticker_interval_mins) + found_missing = False for pair, df in data.items(): if len(df) < expected_frames: + found_missing = True logger.warning('%s has missing frames: expected %s, got %s', pair, expected_frames, len(df)) + return found_missing def load_tickerdata_file( diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 7b13b498a..32db5abd0 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -465,8 +465,8 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: ) min_date, max_date = optimize.get_timeframe(data) caplog.clear() - optimize.validate_backtest_data(data, min_date, max_date, - constants.TICKER_INTERVAL_MINUTES["1m"]) + assert optimize.validate_backtest_data(data, min_date, max_date, + constants.TICKER_INTERVAL_MINUTES["1m"]) assert len(caplog.record_tuples) == 1 assert log_has('UNITTEST/BTC has missing frames: expected 14396, got 13680', caplog.record_tuples) @@ -488,7 +488,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None: min_date, max_date = optimize.get_timeframe(data) caplog.clear() - optimize.validate_backtest_data(data, min_date, max_date, - constants.TICKER_INTERVAL_MINUTES["5m"]) + assert not optimize.validate_backtest_data(data, min_date, max_date, + constants.TICKER_INTERVAL_MINUTES["5m"]) assert len(caplog.record_tuples) == 0 From 3c6d10f03e95b4ede9d63328e4d2f0a65cd2a294 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 Oct 2018 20:05:57 +0200 Subject: [PATCH 6/7] Print missing value count too --- freqtrade/optimize/__init__.py | 7 ++++--- freqtrade/tests/optimize/test_optimize.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 3376d0075..e837a09bd 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -91,10 +91,11 @@ def validate_backtest_data(data: Dict[str, DataFrame], min_date: datetime, expected_frames = int((max_date - min_date).total_seconds() // 60 // ticker_interval_mins) found_missing = False for pair, df in data.items(): - if len(df) < expected_frames: + dflen = len(df) + if dflen < expected_frames: found_missing = True - logger.warning('%s has missing frames: expected %s, got %s', - pair, expected_frames, len(df)) + logger.warning("%s has missing frames: expected %s, got %s, that's %s missing values", + pair, expected_frames, dflen, expected_frames - dflen) return found_missing diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index 32db5abd0..2975a3e4b 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -468,8 +468,9 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None: assert optimize.validate_backtest_data(data, min_date, max_date, constants.TICKER_INTERVAL_MINUTES["1m"]) assert len(caplog.record_tuples) == 1 - assert log_has('UNITTEST/BTC has missing frames: expected 14396, got 13680', - caplog.record_tuples) + assert log_has( + "UNITTEST/BTC has missing frames: expected 14396, got 13680, that's 716 missing values", + caplog.record_tuples) def test_validate_backtest_data(default_conf, mocker, caplog) -> None: @@ -491,4 +492,3 @@ def test_validate_backtest_data(default_conf, mocker, caplog) -> None: assert not optimize.validate_backtest_data(data, min_date, max_date, constants.TICKER_INTERVAL_MINUTES["5m"]) assert len(caplog.record_tuples) == 0 - From 7f9f53248c1aa87a2d7483a9e465d0fc93d68b1e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 18 Oct 2018 20:25:21 +0200 Subject: [PATCH 7/7] Add validate_backtest_data script --- scripts/plot_dataframe.py | 381 -------------------------------------- 1 file changed, 381 deletions(-) delete mode 100755 scripts/plot_dataframe.py diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py deleted file mode 100755 index 68713f296..000000000 --- a/scripts/plot_dataframe.py +++ /dev/null @@ -1,381 +0,0 @@ -#!/usr/bin/env python3 -""" -Script to display when the bot will buy a specific pair - -Mandatory Cli parameters: --p / --pair: pair to examine - -Option but recommended --s / --strategy: strategy to use - - -Optional Cli parameters --d / --datadir: path to pair backtest data ---timerange: specify what timerange of data to use. --l / --live: Live, to download the latest ticker for the pair --db / --db-url: Show trades stored in database - - -Indicators recommended -Row 1: sma, ema3, ema5, ema10, ema50 -Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk - -Example of usage: -> python3 scripts/plot_dataframe.py --pair BTC/EUR -d user_data/data/ --indicators1 sma,ema3 ---indicators2 fastk,fastd -""" -import json -import logging -import sys -from argparse import Namespace -from pathlib import Path -from typing import Dict, List, Any - -import pandas as pd -import plotly.graph_objs as go -import pytz - -from plotly import tools -from plotly.offline import plot - -import freqtrade.optimize as optimize -from freqtrade import persistence -from freqtrade.arguments import Arguments, TimeRange -from freqtrade.exchange import Exchange -from freqtrade.optimize.backtesting import setup_configuration -from freqtrade.persistence import Trade -from freqtrade.strategy.resolver import StrategyResolver - -logger = logging.getLogger(__name__) -_CONF: Dict[str, Any] = {} - -timeZone = pytz.UTC - - -def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: - trades: pd.DataFrame = pd.DataFrame() - if args.db_url: - persistence.init(_CONF) - columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] - - for x in Trade.query.all(): - print("date: {}".format(x.open_date)) - - trades = pd.DataFrame([(t.pair, t.calc_profit(), - t.open_date.replace(tzinfo=timeZone), - t.close_date.replace(tzinfo=timeZone) if t.close_date else None, - t.open_rate, t.close_rate, - t.close_date.timestamp() - t.open_date.timestamp() if t.close_date else None) - for t in Trade.query.filter(Trade.pair.is_(pair)).all()], - columns=columns) - - elif args.exportfilename: - file = Path(args.exportfilename) - # must align with columns in backtest.py - columns = ["pair", "profit", "opents", "closets", "index", "duration", - "open_rate", "close_rate", "open_at_end", "sell_reason"] - with file.open() as f: - data = json.load(f) - trades = pd.DataFrame(data, columns=columns) - trades = trades.loc[trades["pair"] == pair] - if timerange: - if timerange.starttype == 'date': - trades = trades.loc[trades["opents"] >= timerange.startts] - if timerange.stoptype == 'date': - trades = trades.loc[trades["opents"] <= timerange.stopts] - - trades['opents'] = pd.to_datetime(trades['opents'], - unit='s', - utc=True, - infer_datetime_format=True) - trades['closets'] = pd.to_datetime(trades['closets'], - unit='s', - utc=True, - infer_datetime_format=True) - return trades - - -def plot_analyzed_dataframe(args: Namespace) -> None: - """ - Calls analyze() and plots the returned dataframe - :return: None - """ - global _CONF - - # Load the configuration - _CONF.update(setup_configuration(args)) - - print(_CONF) - # Set the pair to audit - pair = args.pair - - if pair is None: - logger.critical('Parameter --pair mandatory;. E.g --pair ETH/BTC') - exit() - - if '/' not in pair: - logger.critical('--pair format must be XXX/YYY') - exit() - - # Set timerange to use - timerange = Arguments.parse_timerange(args.timerange) - - # Load the strategy - try: - strategy = StrategyResolver(_CONF).strategy - exchange = Exchange(_CONF) - except AttributeError: - logger.critical( - 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', - args.strategy - ) - exit() - - # Set the ticker to use - tick_interval = strategy.ticker_interval - - # Load pair tickers - tickers = {} - if args.live: - logger.info('Downloading pair.') - exchange.refresh_tickers([pair], tick_interval) - tickers[pair] = exchange.klines[pair] - else: - tickers = optimize.load_data( - datadir=_CONF.get("datadir"), - pairs=[pair], - ticker_interval=tick_interval, - refresh_pairs=_CONF.get('refresh_pairs', False), - timerange=timerange, - exchange=Exchange(_CONF) - ) - - # No ticker found, or impossible to download - if tickers == {}: - exit() - - # Get trades already made from the DB - trades = load_trades(args, pair, timerange) - - dataframes = strategy.tickerdata_to_dataframe(tickers) - - dataframe = dataframes[pair] - dataframe = strategy.advise_buy(dataframe, {'pair': pair}) - dataframe = strategy.advise_sell(dataframe, {'pair': pair}) - - if len(dataframe.index) > args.plot_limit: - logger.warning('Ticker contained more than %s candles as defined ' - 'with --plot-limit, clipping.', args.plot_limit) - dataframe = dataframe.tail(args.plot_limit) - - trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']] - fig = generate_graph( - pair=pair, - trades=trades, - data=dataframe, - args=args - ) - - plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html'))) - - -def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: - """ - Generate the graph from the data generated by Backtesting or from DB - :param pair: Pair to Display on the graph - :param trades: All trades created - :param data: Dataframe - :param args: sys.argv that contrains the two params indicators1, and indicators2 - :return: None - """ - - # Define the graph - fig = tools.make_subplots( - rows=3, - cols=1, - shared_xaxes=True, - row_width=[1, 1, 4], - vertical_spacing=0.0001, - ) - fig['layout'].update(title=pair) - fig['layout']['yaxis1'].update(title='Price') - fig['layout']['yaxis2'].update(title='Volume') - fig['layout']['yaxis3'].update(title='Other') - - # Common information - candles = go.Candlestick( - x=data.date, - open=data.open, - high=data.high, - low=data.low, - close=data.close, - name='Price' - ) - - df_buy = data[data['buy'] == 1] - buys = go.Scattergl( - x=df_buy.date, - y=df_buy.close, - mode='markers', - name='buy', - marker=dict( - symbol='triangle-up-dot', - size=9, - line=dict(width=1), - color='green', - ) - ) - df_sell = data[data['sell'] == 1] - sells = go.Scattergl( - x=df_sell.date, - y=df_sell.close, - mode='markers', - name='sell', - marker=dict( - symbol='triangle-down-dot', - size=9, - line=dict(width=1), - color='red', - ) - ) - - trade_buys = go.Scattergl( - x=trades["opents"], - y=trades["open_rate"], - mode='markers', - name='trade_buy', - marker=dict( - symbol='square-open', - size=11, - line=dict(width=2), - color='green' - ) - ) - trade_sells = go.Scattergl( - x=trades["closets"], - y=trades["close_rate"], - mode='markers', - name='trade_sell', - marker=dict( - symbol='square-open', - size=11, - line=dict(width=2), - color='red' - ) - ) - - # Row 1 - fig.append_trace(candles, 1, 1) - - if 'bb_lowerband' in data and 'bb_upperband' in data: - bb_lower = go.Scatter( - x=data.date, - y=data.bb_lowerband, - name='BB lower', - line={'color': 'rgba(255,255,255,0)'}, - ) - bb_upper = go.Scatter( - x=data.date, - y=data.bb_upperband, - name='BB upper', - fill="tonexty", - fillcolor="rgba(0,176,246,0.2)", - line={'color': 'rgba(255,255,255,0)'}, - ) - fig.append_trace(bb_lower, 1, 1) - fig.append_trace(bb_upper, 1, 1) - - fig = generate_row(fig=fig, row=1, raw_indicators=args.indicators1, data=data) - fig.append_trace(buys, 1, 1) - fig.append_trace(sells, 1, 1) - fig.append_trace(trade_buys, 1, 1) - fig.append_trace(trade_sells, 1, 1) - - # Row 2 - volume = go.Bar( - x=data['date'], - y=data['volume'], - name='Volume' - ) - fig.append_trace(volume, 2, 1) - - # Row 3 - fig = generate_row(fig=fig, row=3, raw_indicators=args.indicators2, data=data) - - return fig - - -def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots: - """ - Generator all the indicator selected by the user for a specific row - """ - for indicator in raw_indicators.split(','): - if indicator in data: - scattergl = go.Scattergl( - x=data['date'], - y=data[indicator], - name=indicator - ) - fig.append_trace(scattergl, row, 1) - else: - logger.info( - 'Indicator "%s" ignored. Reason: This indicator is not found ' - 'in your strategy.', - indicator - ) - - return fig - - -def plot_parse_args(args: List[str]) -> Namespace: - """ - Parse args passed to the script - :param args: Cli arguments - :return: args: Array with all arguments - """ - arguments = Arguments(args, 'Graph dataframe') - arguments.scripts_options() - arguments.parser.add_argument( - '--indicators1', - help='Set indicators from your strategy you want in the first row of the graph. Separate ' - 'them with a coma. E.g: ema3,ema5 (default: %(default)s)', - type=str, - default='sma,ema3,ema5', - dest='indicators1', - ) - - arguments.parser.add_argument( - '--indicators2', - help='Set indicators from your strategy you want in the third row of the graph. Separate ' - 'them with a coma. E.g: fastd,fastk (default: %(default)s)', - type=str, - default='macd', - dest='indicators2', - ) - arguments.parser.add_argument( - '--plot-limit', - help='Specify tick limit for plotting - too high values cause huge files - ' - 'Default: %(default)s', - dest='plot_limit', - default=750, - type=int, - ) - arguments.common_args_parser() - arguments.optimizer_shared_options(arguments.parser) - arguments.backtesting_options(arguments.parser) - return arguments.parse_args() - - -def main(sysargv: List[str]) -> None: - """ - This function will initiate the bot and start the trading loop. - :return: None - """ - logger.info('Starting Plot Dataframe') - plot_analyzed_dataframe( - plot_parse_args(sysargv) - ) - - -if __name__ == '__main__': - main(sys.argv[1:])