From e7d043974199ab041ccfd44111c73d330be62f0e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Jul 2018 23:00:50 +0200 Subject: [PATCH 01/51] Add new arguments --- freqtrade/arguments.py | 8 ++++++++ freqtrade/configuration.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 022a2c739..042eeedf1 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -142,6 +142,14 @@ class Arguments(object): action='store_true', dest='refresh_pairs', ) + parser.add_argument( + '--strategy-list', + help='Provide a commaseparated list of strategies to backtest ' + 'Please note that ticker-interval needs to be set either in config ' + 'or via command line', + nargs='+', + dest='strategy_list', + ) parser.add_argument( '--export', help='export backtest results, argument are: trades\ diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index dcc6e4332..aa452c79d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -187,6 +187,14 @@ class Configuration(object): config.update({'refresh_pairs': True}) logger.info('Parameter -r/--refresh-pairs-cached detected ...') + if 'strategy_list' in self.args and self.args.strategy_list: + config.update({'strategy_list': self.args.strategy_list}) + logger.info('using strategy list of %s Strategies', len(self.args.strategy_list)) + + if 'ticker_interval' in self.args and self.args.ticker_interval: + config.update({'ticker_interval': self.args.ticker_interval}) + logger.info('Overriding ticker interval with Command line argument') + # If --export is used we add it to the configuration if 'export' in self.args and self.args.export: config.update({'export': self.args.export}) From 56046b3cb39c16ba8a43e43cca88de2d5ecfa51c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Jul 2018 23:01:52 +0200 Subject: [PATCH 02/51] Add strategylist option to backtesting --- freqtrade/optimize/backtesting.py | 126 +++++++++++++++++------------- 1 file changed, 71 insertions(+), 55 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 593af619c..4146c25dd 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,6 +6,7 @@ This module contains the backtesting logic import logging import operator from argparse import Namespace +from copy import deepcopy from datetime import datetime, timedelta from typing import Any, Dict, List, NamedTuple, Optional, Tuple @@ -54,11 +55,6 @@ class Backtesting(object): """ def __init__(self, config: Dict[str, Any]) -> None: self.config = config - self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.ticker_interval = self.strategy.ticker_interval - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.advise_buy = self.strategy.advise_buy - self.advise_sell = self.strategy.advise_sell # Reset keys for backtesting self.config['exchange']['key'] = '' @@ -279,6 +275,19 @@ class Backtesting(object): pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + strategylist: List[IStrategy] = [] + if self.config.get('strategy_list', None): + # Force one interval + self.ticker_interval = self.config.get('ticker_interval') + for strat in self.config.get('strategy_list'): + stratconf = deepcopy(self.config) + stratconf['strategy'] = strat + s = StrategyResolver(stratconf).strategy + strategylist.append(s) + + else: + # only one strategy + strategylist.append(StrategyResolver(self.config).strategy) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') @@ -308,61 +317,68 @@ class Backtesting(object): logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 - preprocessed = self.tickerdata_to_dataframe(data) + for strat in strategylist: + self.strategy = strat + self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe + self.populate_buy_trend = self.strategy.populate_buy_trend + self.populate_sell_trend = self.strategy.populate_sell_trend - # Print timeframe - min_date, max_date = self.get_timeframe(preprocessed) - logger.info( - 'Measuring data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days - ) + # need to reprocess data every time to populate signals + preprocessed = self.tickerdata_to_dataframe(data) - # Execute backtest and print results - results = self.backtest( - { - 'stake_amount': self.config.get('stake_amount'), - 'processed': preprocessed, - 'max_open_trades': max_open_trades, - 'position_stacking': self.config.get('position_stacking', False), - } - ) - - if self.config.get('export', False): - self._store_backtest_result(self.config.get('exportfilename'), results) - - logger.info( - '\n' + '=' * 49 + - ' BACKTESTING REPORT ' + - '=' * 50 + '\n' - '%s', - self._generate_text_table( - data, - results + # Print timeframe + min_date, max_date = self.get_timeframe(preprocessed) + logger.info( + 'Measuring data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days ) - ) - # logger.info( - # results[['sell_reason']].groupby('sell_reason').count() - # ) - logger.info( - '\n' + - ' SELL READON STATS '.center(119, '=') + - '\n%s \n', - self._generate_text_table_sell_reason(data, results) - - ) - - logger.info( - '\n' + - ' LEFT OPEN TRADES REPORT '.center(119, '=') + - '\n%s', - self._generate_text_table( - data, - results.loc[results.open_at_end] + # Execute backtest and print results + results = self.backtest( + { + 'stake_amount': self.config.get('stake_amount'), + 'processed': preprocessed, + 'max_open_trades': max_open_trades, + 'position_stacking': self.config.get('position_stacking', False), + } + ) + + if self.config.get('export', False): + self._store_backtest_result(self.config.get('exportfilename'), results) + + logger.info( + '\n' + '=' * 49 + + ' BACKTESTING REPORT ' + + '=' * 50 + '\n' + '%s', + self._generate_text_table( + data, + results + ) + ) + # logger.info( + # results[['sell_reason']].groupby('sell_reason').count() + # ) + + logger.info( + '\n' + + ' SELL READON STATS '.center(119, '=') + + '\n%s \n', + self._generate_text_table_sell_reason(data, results) + + ) + + logger.info( + '\n' + + ' LEFT OPEN TRADES REPORT '.center(119, '=') + + '\n%s', + self._generate_text_table( + data, + results.loc[results.open_at_end] + ) ) - ) def setup_configuration(args: Namespace) -> Dict[str, Any]: From 9a42aac0f24e82694e783d146c6069bb24663abe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 06:40:39 +0200 Subject: [PATCH 03/51] Add testcase for --strategylist --- freqtrade/configuration.py | 2 +- freqtrade/tests/test_arguments.py | 8 +++- freqtrade/tests/test_configuration.py | 55 +++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index aa452c79d..3da432b1d 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -189,7 +189,7 @@ class Configuration(object): if 'strategy_list' in self.args and self.args.strategy_list: config.update({'strategy_list': self.args.strategy_list}) - logger.info('using strategy list of %s Strategies', len(self.args.strategy_list)) + logger.info('Using strategy list of %s Strategies', len(self.args.strategy_list)) if 'ticker_interval' in self.args and self.args.ticker_interval: config.update({'ticker_interval': self.args.ticker_interval}) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 79bd0254b..e09aeb1df 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -132,7 +132,11 @@ def test_parse_args_backtesting_custom() -> None: 'backtesting', '--live', '--ticker-interval', '1m', - '--refresh-pairs-cached'] + '--refresh-pairs-cached', + '--strategy-list', + 'DefaultStrategy', + 'TestStrategy' + ] call_args = Arguments(args, '').get_parsed_arg() assert call_args.config == 'test_conf.json' assert call_args.live is True @@ -141,6 +145,8 @@ def test_parse_args_backtesting_custom() -> None: assert call_args.func is not None assert call_args.ticker_interval == '1m' assert call_args.refresh_pairs is True + assert type(call_args.strategy_list) is list + assert len(call_args.strategy_list) == 2 def test_parse_args_hyperopt_custom() -> None: diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index e48553bdf..bf41aab83 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -292,6 +292,61 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non ) +def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> None: + """ + Test setup_configuration() function + """ + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(default_conf) + )) + + arglist = [ + '--config', 'config.json', + 'backtesting', + '--ticker-interval', '1m', + '--export', '/bar/foo', + '--strategy-list', + 'DefaultStrategy', + 'TestStrategy' + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + assert 'max_open_trades' in config + assert 'stake_currency' in config + assert 'stake_amount' in config + assert 'exchange' in config + assert 'pair_whitelist' in config['exchange'] + assert 'datadir' in config + assert log_has( + 'Using data folder: {} ...'.format(config['datadir']), + caplog.record_tuples + ) + assert 'ticker_interval' in config + assert log_has('Parameter -i/--ticker-interval detected ...', caplog.record_tuples) + assert log_has( + 'Using ticker_interval: 1m ...', + caplog.record_tuples + ) + + assert 'strategy_list' in config + assert log_has('Using strategy list of 2 Strategies', caplog.record_tuples) + + assert 'position_stacking' not in config + + assert 'use_max_market_positions' not in config + + assert 'timerange' not in config + + assert 'export' in config + assert log_has( + 'Parameter --export detected: {} ...'.format(config['export']), + caplog.record_tuples + ) + + def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) From 65aaa3dffdea2ddae22795cdc202cccb1dbca56d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 06:54:33 +0200 Subject: [PATCH 04/51] Extract backtest strategy setting --- freqtrade/optimize/backtesting.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4146c25dd..14a66f2ac 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -65,6 +65,16 @@ class Backtesting(object): self.exchange = Exchange(self.config) self.fee = self.exchange.get_fee() + def set_strategy(self, strategy): + """ + Load strategy into backtesting + """ + self.strategy = strategy + self.ticker_interval = self.config.get('ticker_interval') + self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe + self.populate_buy_trend = strategy.populate_buy_trend + self.populate_sell_trend = strategy.populate_sell_trend + @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ @@ -288,6 +298,7 @@ class Backtesting(object): else: # only one strategy strategylist.append(StrategyResolver(self.config).strategy) + self.set_strategy(strategylist[0]) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') @@ -316,12 +327,11 @@ class Backtesting(object): else: logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 + all_results = {} for strat in strategylist: - self.strategy = strat - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.populate_buy_trend = self.strategy.populate_buy_trend - self.populate_sell_trend = self.strategy.populate_sell_trend + logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) + self.set_strategy(strat) # need to reprocess data every time to populate signals preprocessed = self.tickerdata_to_dataframe(data) @@ -336,7 +346,7 @@ class Backtesting(object): ) # Execute backtest and print results - results = self.backtest( + all_results[self.strategy.get_strategy_name()] = self.backtest( { 'stake_amount': self.config.get('stake_amount'), 'processed': preprocessed, @@ -345,14 +355,16 @@ class Backtesting(object): } ) + for strategy, results in all_results.items(): + if self.config.get('export', False): self._store_backtest_result(self.config.get('exportfilename'), results) + logger.info("\nResult for strategy %s", strategy) logger.info( - '\n' + '=' * 49 + - ' BACKTESTING REPORT ' + - '=' * 50 + '\n' - '%s', + '\n' + + ' BACKTESTING REPORT '.center(119, '=') + + '\n%s', self._generate_text_table( data, results From 5f2e92ec5c7329ba3fe5b53ca4b5dd65332c7a2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 07:00:58 +0200 Subject: [PATCH 05/51] Refactor backtesting --- freqtrade/optimize/backtesting.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 14a66f2ac..a9121a3d0 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -53,6 +53,7 @@ class Backtesting(object): backtesting = Backtesting(config) backtesting.start() """ + def __init__(self, config: Dict[str, Any]) -> None: self.config = config @@ -62,10 +63,14 @@ class Backtesting(object): self.config['exchange']['password'] = '' self.config['exchange']['uid'] = '' self.config['dry_run'] = True + if not self.config.get('strategy_list'): + # In Single strategy mode, load strategy here to avoid problems with hyperopt + self._set_strategy(StrategyResolver(self.config).strategy) + self.exchange = Exchange(self.config) self.fee = self.exchange.get_fee() - def set_strategy(self, strategy): + def _set_strategy(self, strategy): """ Load strategy into backtesting """ @@ -297,8 +302,7 @@ class Backtesting(object): else: # only one strategy - strategylist.append(StrategyResolver(self.config).strategy) - self.set_strategy(strategylist[0]) + strategylist.append(self.strategy) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') @@ -331,7 +335,7 @@ class Backtesting(object): for strat in strategylist: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) - self.set_strategy(strat) + self._set_strategy(strat) # need to reprocess data every time to populate signals preprocessed = self.tickerdata_to_dataframe(data) From 644f729aeabf42cd2ac7da45d43881b6bfdebe96 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 07:41:38 +0200 Subject: [PATCH 06/51] Refactor strategy loading to __init__ --- freqtrade/optimize/backtesting.py | 34 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a9121a3d0..ffd89635a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -63,9 +63,22 @@ class Backtesting(object): self.config['exchange']['password'] = '' self.config['exchange']['uid'] = '' self.config['dry_run'] = True - if not self.config.get('strategy_list'): - # In Single strategy mode, load strategy here to avoid problems with hyperopt - self._set_strategy(StrategyResolver(self.config).strategy) + self.strategylist: List[IStrategy] = [] + if self.config.get('strategy_list', None): + # Force one interval + self.ticker_interval = self.config.get('ticker_interval') + for strat in self.config.get('strategy_list'): + stratconf = deepcopy(self.config) + stratconf['strategy'] = strat + self.strategylist.append(StrategyResolver(stratconf).strategy) + + else: + # only one strategy + strat = StrategyResolver(self.config).strategy + + self.strategylist.append(StrategyResolver(self.config).strategy) + # Load one strategy + self._set_strategy(self.strategylist[0]) self.exchange = Exchange(self.config) self.fee = self.exchange.get_fee() @@ -290,19 +303,6 @@ class Backtesting(object): pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - strategylist: List[IStrategy] = [] - if self.config.get('strategy_list', None): - # Force one interval - self.ticker_interval = self.config.get('ticker_interval') - for strat in self.config.get('strategy_list'): - stratconf = deepcopy(self.config) - stratconf['strategy'] = strat - s = StrategyResolver(stratconf).strategy - strategylist.append(s) - - else: - # only one strategy - strategylist.append(self.strategy) if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') @@ -333,7 +333,7 @@ class Backtesting(object): max_open_trades = 0 all_results = {} - for strat in strategylist: + for strat in self.strategylist: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) self._set_strategy(strat) From bd3563df6738409d7b0e92e33d879d193a81b16b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 07:55:59 +0200 Subject: [PATCH 07/51] Add test for new functionality --- freqtrade/optimize/backtesting.py | 4 +- freqtrade/tests/optimize/test_backtesting.py | 63 +++++++++++++++++--- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ffd89635a..0bd76b2c4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -66,8 +66,8 @@ class Backtesting(object): self.strategylist: List[IStrategy] = [] if self.config.get('strategy_list', None): # Force one interval - self.ticker_interval = self.config.get('ticker_interval') - for strat in self.config.get('strategy_list'): + self.ticker_interval = str(self.config.get('ticker_interval')) + for strat in list(self.config['strategy_list']): stratconf = deepcopy(self.config) stratconf['strategy'] = strat self.strategylist.append(StrategyResolver(stratconf).strategy) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 5d121d27c..d91781ffc 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -686,15 +686,6 @@ def test_backtest_start_live(default_conf, mocker, caplog): read_data=json.dumps(default_conf) )) - args = MagicMock() - args.ticker_interval = 1 - args.level = 10 - args.live = True - args.datadir = None - args.export = None - args.strategy = 'DefaultStrategy' - args.timerange = '-100' # needed due to MagicMock malleability - args = [ '--config', 'config.json', '--strategy', 'DefaultStrategy', @@ -725,3 +716,57 @@ def test_backtest_start_live(default_conf, mocker, caplog): for line in exists: assert log_has(line, caplog.record_tuples) + + +def test_backtest_start_multi_strat(default_conf, mocker, caplog): + conf = deepcopy(default_conf) + conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] + mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', + new=lambda s, n, i: _load_pair_as_ticks(n, i)) + patch_exchange(mocker) + backtestmock = MagicMock() + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + gen_table_mock = MagicMock() + mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', gen_table_mock) + mocker.patch('freqtrade.configuration.open', mocker.mock_open( + read_data=json.dumps(conf) + )) + + args = [ + '--config', 'config.json', + '--datadir', 'freqtrade/tests/testdata', + 'backtesting', + '--ticker-interval', '1m', + '--live', + '--timerange', '-100', + '--enable-position-stacking', + '--disable-max-market-positions', + '--strategy-list', + 'DefaultStrategy', + 'TestStrategy', + ] + args = get_args(args) + start(args) + # 2 backtests, 4 tables + assert backtestmock.call_count == 2 + assert gen_table_mock.call_count == 4 + + # check the logs, that will contain the backtest result + exists = [ + 'Parameter -i/--ticker-interval detected ...', + 'Using ticker_interval: 1m ...', + 'Parameter -l/--live detected ...', + 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', + 'Parameter --timerange detected: -100 ...', + 'Using data folder: freqtrade/tests/testdata ...', + 'Using stake_currency: BTC ...', + 'Using stake_amount: 0.001 ...', + 'Downloading data for all pairs in whitelist ...', + 'Measuring data from 2017-11-14T19:31:00+00:00 up to 2017-11-14T22:58:00+00:00 (0 days)..', + 'Parameter --enable-position-stacking detected ...', + 'Running backtesting for Strategy DefaultStrategy', + 'Running backtesting for Strategy TestStrategy', + ] + + for line in exists: + assert log_has(line, caplog.record_tuples) From a57a2f4a750b52dcf5871e5aec329a34c5d60f32 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 21:55:47 +0200 Subject: [PATCH 08/51] Store backtest-result in different vars --- freqtrade/arguments.py | 4 +++- freqtrade/optimize/backtesting.py | 12 ++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 042eeedf1..501c1784f 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -146,7 +146,9 @@ class Arguments(object): '--strategy-list', help='Provide a commaseparated list of strategies to backtest ' 'Please note that ticker-interval needs to be set either in config ' - 'or via command line', + 'or via command line. When using this together with --export trades, ' + 'the strategy-name is injected into the filename ' + '(so backtest-data.json becomes backtest-data-DefaultStrategy.json', nargs='+', dest='strategy_list', ) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0bd76b2c4..69d48b027 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -8,6 +8,7 @@ import operator from argparse import Namespace from copy import deepcopy from datetime import datetime, timedelta +from pathlib import Path from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow @@ -156,7 +157,8 @@ class Backtesting(object): tabular_data.append([reason.value, count]) return tabulate(tabular_data, headers=headers, tablefmt="pipe") - def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: + def _store_backtest_result(self, recordfilename: str, results: DataFrame, + strategyname: Optional[str] = None) -> None: records = [(t.pair, t.profit_percent, t.open_time.timestamp(), t.close_time.timestamp(), t.open_index - 1, t.trade_duration, @@ -164,6 +166,11 @@ class Backtesting(object): for index, t in results.iterrows()] if records: + if strategyname: + # Inject strategyname to filename + recname = Path(recordfilename) + recordfilename = str(Path.joinpath( + recname.parent, f'{recname.stem}-{strategyname}').with_suffix(recname.suffix)) logger.info('Dumping backtest results to %s', recordfilename) file_dump_json(recordfilename, records) @@ -362,7 +369,8 @@ class Backtesting(object): for strategy, results in all_results.items(): if self.config.get('export', False): - self._store_backtest_result(self.config.get('exportfilename'), results) + self._store_backtest_result(self.config['exportfilename'], results, + strategy if len(self.strategylist) > 1 else None) logger.info("\nResult for strategy %s", strategy) logger.info( From a8b55b8989387f083250b0b8bc7dbdefd05e4d3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 28 Jul 2018 22:00:12 +0200 Subject: [PATCH 09/51] Add test for strategy-name injection --- freqtrade/tests/optimize/test_backtesting.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index d91781ffc..311fe7da4 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -654,6 +654,18 @@ def test_backtest_record(default_conf, fee, mocker): records = records[0] # Ensure records are of correct type assert len(records) == 4 + + # reset test to test with strategy name + names = [] + records = [] + backtesting._store_backtest_result("backtest-result.json", results, "DefStrat") + assert len(results) == 4 + # Assert file_dump_json was only called once + assert names == ['backtest-result-DefStrat.json'] + records = records[0] + # Ensure records are of correct type + assert len(records) == 4 + # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) # Below follows just a typecheck of the schema/type of trade-records oix = None From 4ea6780153ae4ab1ddab61c4d4ea6eb636a5dc7a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 09:51:45 +0200 Subject: [PATCH 10/51] Update documentation with --strategy-list --- docs/backtesting.md | 19 ++++++++++++++++++- docs/bot-usage.md | 33 ++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 766875970..2d53303c5 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -151,7 +151,7 @@ cp freqtrade/tests/testdata/pairs.json user_data/data/binance Then run: ```bash -python scripts/download_backtest_data --exchange binance +python scripts/download_backtest_data.py --exchange binance ``` This will download ticker data for all the currency pairs you defined in `pairs.json`. @@ -238,6 +238,23 @@ On the other hand, if you set a too high `minimal_roi` like `"0": 0.55` profit. Hence, keep in mind that your performance is a mix of your strategies, your configuration, and the crypto-currency you have set up. +## Backtesting multiple strategies + +To backtest multiple strategies, a list of Strategies can be provided. + +This is limited to 1 ticker-interval per run, however, data is only loaded once from disk so if you have multiple +strategies you'd like to compare, this should give a nice runtime boost. + +All listed Strategies need to be in the same folder. + +``` bash +freqtrade backtesting --timerange 20180401-20180410 --ticker-interval 5m --strategy-list Strategy001 Strategy002 --export trades +``` + +This will save the results to `user_data/backtest_data/backtest-result-.json`, injecting the strategy-name into the target filename. +It will also output all results one after the other, so make sure to scroll up. + + ## Next step Great, your strategy is profitable. What if the bot can give your the diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 4e479adac..83a8ee833 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -1,13 +1,15 @@ # Bot usage -This page explains the difference parameters of the bot and how to run -it. + +This page explains the difference parameters of the bot and how to run it. ## Table of Contents + - [Bot commands](#bot-commands) - [Backtesting commands](#backtesting-commands) - [Hyperopt commands](#hyperopt-commands) ## Bot commands + ``` usage: freqtrade [-h] [-v] [--version] [-c PATH] [-d PATH] [-s NAME] [--strategy-path PATH] [--dynamic-whitelist [INT]] @@ -41,6 +43,7 @@ optional arguments: ``` ### How to use a different config file? + The bot allows you to select which config file you want to use. Per default, the bot will load the file `./config.json` @@ -49,6 +52,7 @@ python3 ./freqtrade/main.py -c path/far/far/away/config.json ``` ### How to use --strategy? + This parameter will allow you to load your custom strategy class. Per default without `--strategy` or `-s` the bot will load the `DefaultStrategy` included with the bot (`freqtrade/strategy/default_strategy.py`). @@ -60,6 +64,7 @@ To load a strategy, simply pass the class name (e.g.: `CustomStrategy`) in this **Example:** In `user_data/strategies` you have a file `my_awesome_strategy.py` which has a strategy class called `AwesomeStrategy` to load it: + ```bash python3 ./freqtrade/main.py --strategy AwesomeStrategy ``` @@ -70,6 +75,7 @@ message the reason (File not found, or errors in your code). Learn more about strategy file in [optimize your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md). ### How to use --strategy-path? + This parameter allows you to add an additional strategy lookup path, which gets checked before the default locations (The passed path must be a folder!): ```bash @@ -77,21 +83,25 @@ python3 ./freqtrade/main.py --strategy AwesomeStrategy --strategy-path /some/fol ``` #### How to install a strategy? + This is very simple. Copy paste your strategy file into the folder `user_data/strategies` or use `--strategy-path`. And voila, the bot is ready to use it. ### How to use --dynamic-whitelist? + Per default `--dynamic-whitelist` will retrieve the 20 currencies based on BaseVolume. This value can be changed when you run the script. **By Default** Get the 20 currencies based on BaseVolume. + ```bash python3 ./freqtrade/main.py --dynamic-whitelist ``` **Customize the number of currencies to retrieve** Get the 30 currencies based on BaseVolume. + ```bash python3 ./freqtrade/main.py --dynamic-whitelist 30 ``` @@ -102,6 +112,7 @@ negative value (e.g -2), `--dynamic-whitelist` will use the default value (20). ### How to use --db-url? + When you run the bot in Dry-run mode, per default no transactions are stored in a database. If you want to store your bot actions in a DB using `--db-url`. This can also be used to specify a custom database @@ -111,14 +122,14 @@ in production mode. Example command: python3 ./freqtrade/main.py -c config.json --db-url sqlite:///tradesv3.dry_run.sqlite ``` - ## Backtesting commands Backtesting also uses the config specified via `-c/--config`. ``` -usage: main.py backtesting [-h] [-i TICKER_INTERVAL] [--eps] [--dmmp] +usage: freqtrade backtesting [-h] [-i TICKER_INTERVAL] [--eps] [--dmmp] [--timerange TIMERANGE] [-l] [-r] + [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] [--export EXPORT] [--export-filename PATH] optional arguments: @@ -139,6 +150,13 @@ optional arguments: refresh the pairs files in tests/testdata with the latest data from the exchange. Use it if you want to run your backtesting with up-to-date data. + --strategy-list STRATEGY_LIST [STRATEGY_LIST ...] + Provide a commaseparated list of strategies to + backtest Please note that ticker-interval needs to be + set either in config or via command line. When using + this together with --export trades, the strategy-name + is injected into the filename (so backtest-data.json + becomes backtest-data-DefaultStrategy.json --export EXPORT export backtest results, argument are: trades Example --export=trades --export-filename PATH @@ -151,6 +169,7 @@ optional arguments: ``` ### How to use --refresh-pairs-cached parameter? + The first time your run Backtesting, it will take the pairs you have set in your config file and download data from Bittrex. @@ -162,7 +181,6 @@ to come back to the previous version.** To test your strategy with latest data, we recommend continuing using the parameter `-l` or `--live`. - ## Hyperopt commands To optimize your strategy, you can use hyperopt parameter hyperoptimization @@ -194,10 +212,11 @@ optional arguments: ``` ## A parameter missing in the configuration? + All parameters for `main.py`, `backtesting`, `hyperopt` are referenced in [misc.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/misc.py#L84) ## Next step -The optimal strategy of the bot will change with time depending of the -market trends. The next step is to + +The optimal strategy of the bot will change with time depending of the market trends. The next step is to [optimize your bot](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md). From 5125076f5d9a809559da3f0717cf553d1fbd6c96 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 10:05:16 +0200 Subject: [PATCH 11/51] Fix typo --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 69d48b027..6e242ac1a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -388,7 +388,7 @@ class Backtesting(object): logger.info( '\n' + - ' SELL READON STATS '.center(119, '=') + + ' SELL REASON STATS '.center(119, '=') + '\n%s \n', self._generate_text_table_sell_reason(data, results) From 028589abd273f23cf708fc77430775c6661bdbfe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 13:07:11 +0200 Subject: [PATCH 12/51] Add strategy summary table --- freqtrade/optimize/backtesting.py | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6e242ac1a..6f571ae27 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -157,6 +157,30 @@ class Backtesting(object): tabular_data.append([reason.value, count]) return tabulate(tabular_data, headers=headers, tablefmt="pipe") + def _generate_text_table_strategy(self, all_results: dict) -> str: + """ + Generate summary table per strategy + """ + stake_currency = str(self.config.get('stake_currency')) + + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f') + tabular_data = [] + headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %', + 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] + for strategy, results in all_results.items(): + tabular_data.append([ + strategy, + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_percent.sum() * 100.0, + results.profit_abs.sum(), + str(timedelta( + minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', + len(results[results.profit_abs > 0]), + len(results[results.profit_abs < 0]) + ]) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") + def _store_backtest_result(self, recordfilename: str, results: DataFrame, strategyname: Optional[str] = None) -> None: @@ -404,6 +428,15 @@ class Backtesting(object): ) ) + if len(all_results) > 1: + # Print Strategy summary table + logger.info( + '\n' + + ' Strategy Summary'.center(119, '=') + + '\n%s\n\nFor more details, please look at the detail tables above', + self._generate_text_table_strategy(all_results) + ) + def setup_configuration(args: Namespace) -> Dict[str, Any]: """ From 765d1c769c9552840d8fd6679dd3788b5e5ddd61 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 13:07:30 +0200 Subject: [PATCH 13/51] Add test for stratgy summary table --- freqtrade/tests/optimize/test_backtesting.py | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 311fe7da4..02f16be85 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -406,6 +406,50 @@ def test_generate_text_table_sell_reason(default_conf, mocker): data={'ETH/BTC': {}}, results=results) == result_str +def test_generate_text_table_strategyn(default_conf, mocker): + """ + Test Backtesting.generate_text_table_sell_reason() method + """ + patch_exchange(mocker) + backtesting = Backtesting(default_conf) + results = {} + results['ETH/BTC'] = pd.DataFrame( + { + 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], + 'profit_percent': [0.1, 0.2, 0.3], + 'profit_abs': [0.2, 0.4, 0.5], + 'trade_duration': [10, 30, 10], + 'profit': [2, 0, 0], + 'loss': [0, 0, 1], + 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + } + ) + results['LTC/BTC'] = pd.DataFrame( + { + 'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'], + 'profit_percent': [0.4, 0.2, 0.3], + 'profit_abs': [0.4, 0.4, 0.5], + 'trade_duration': [15, 30, 15], + 'profit': [4, 1, 0], + 'loss': [0, 0, 1], + 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + } + ) + + result_str = ( + '| Strategy | buy count | avg profit % | cum profit % ' + '| total profit BTC | avg duration | profit | loss |\n' + '|:-----------|------------:|---------------:|---------------:' + '|-------------------:|:---------------|---------:|-------:|\n' + '| ETH/BTC | 3 | 20.00 | 60.00 ' + '| 1.10000000 | 0:17:00 | 3 | 0 |\n' + '| LTC/BTC | 3 | 30.00 | 90.00 ' + '| 1.30000000 | 0:20:00 | 3 | 0 |' + ) + print(backtesting._generate_text_table_strategy(all_results=results)) + assert backtesting._generate_text_table_strategy(all_results=results) == result_str + + def test_backtesting_start(default_conf, mocker, caplog) -> None: def get_timeframe(input1, input2): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) @@ -740,6 +784,9 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) gen_table_mock = MagicMock() mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', gen_table_mock) + gen_strattable_mock = MagicMock() + mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table_strategy', + gen_strattable_mock) mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(conf) )) @@ -762,6 +809,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): # 2 backtests, 4 tables assert backtestmock.call_count == 2 assert gen_table_mock.call_count == 4 + assert gen_strattable_mock.call_count == 1 # check the logs, that will contain the backtest result exists = [ From c648e2acfcad9f3b0af714d3a7105d23d7ffe64a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 13:13:03 +0200 Subject: [PATCH 14/51] Adjust documentation to strategy table --- docs/backtesting.md | 10 +++++++++- freqtrade/optimize/backtesting.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 2d53303c5..cc8ecd6c7 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -252,8 +252,16 @@ freqtrade backtesting --timerange 20180401-20180410 --ticker-interval 5m --strat ``` This will save the results to `user_data/backtest_data/backtest-result-.json`, injecting the strategy-name into the target filename. -It will also output all results one after the other, so make sure to scroll up. +There will be an additional table comparing win/losses of the different strategies (identical to the "Total" row in the first table). +Detailed output for all strategies one after the other will be available, so make sure to scroll up. +``` +=================================================== Strategy Summary ==================================================== +| Strategy | buy count | avg profit % | cum profit % | total profit ETH | avg duration | profit | loss | +|:-----------|------------:|---------------:|---------------:|-------------------:|:----------------|---------:|-------:| +| Strategy1 | 19 | -0.76 | -14.39 | -0.01440287 | 15:48:00 | 15 | 4 | +| Strategy2 | 6 | -2.73 | -16.40 | -0.01641299 | 1 day, 14:12:00 | 3 | 3 | +``` ## Next step diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6f571ae27..067e7bdca 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -432,7 +432,7 @@ class Backtesting(object): # Print Strategy summary table logger.info( '\n' + - ' Strategy Summary'.center(119, '=') + + ' Strategy Summary '.center(119, '=') + '\n%s\n\nFor more details, please look at the detail tables above', self._generate_text_table_strategy(all_results) ) From 76fbb89a03b84bcb35cfcb585af40d8fcc3e4674 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Jul 2018 19:41:39 +0200 Subject: [PATCH 15/51] use print for backtest results to avoid odd newline-handling --- freqtrade/optimize/backtesting.py | 47 ++++++++----------------------- 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 067e7bdca..53071efaf 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -396,46 +396,21 @@ class Backtesting(object): self._store_backtest_result(self.config['exportfilename'], results, strategy if len(self.strategylist) > 1 else None) - logger.info("\nResult for strategy %s", strategy) - logger.info( - '\n' + - ' BACKTESTING REPORT '.center(119, '=') + - '\n%s', - self._generate_text_table( - data, - results - ) - ) - # logger.info( - # results[['sell_reason']].groupby('sell_reason').count() - # ) + print(f"Result for strategy {strategy}") + print(' BACKTESTING REPORT '.center(119, '=')) + print(self._generate_text_table(data, results)) - logger.info( - '\n' + - ' SELL REASON STATS '.center(119, '=') + - '\n%s \n', - self._generate_text_table_sell_reason(data, results) - - ) - - logger.info( - '\n' + - ' LEFT OPEN TRADES REPORT '.center(119, '=') + - '\n%s', - self._generate_text_table( - data, - results.loc[results.open_at_end] - ) - ) + print(' SELL REASON STATS '.center(119, '=')) + print(self._generate_text_table_sell_reason(data, results)) + print(' LEFT OPEN TRADES REPORT '.center(119, '=')) + print(self._generate_text_table(data, results.loc[results.open_at_end])) + print() if len(all_results) > 1: # Print Strategy summary table - logger.info( - '\n' + - ' Strategy Summary '.center(119, '=') + - '\n%s\n\nFor more details, please look at the detail tables above', - self._generate_text_table_strategy(all_results) - ) + print(' Strategy Summary '.center(119, '=')) + print(self._generate_text_table_strategy(all_results)) + print('\nFor more details, please look at the detail tables above') def setup_configuration(args: Namespace) -> Dict[str, Any]: From 40ee86b3579b35cd69e20766d3e7f1b869d36d86 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Jul 2018 21:08:03 +0200 Subject: [PATCH 16/51] Adapt after rebase --- freqtrade/optimize/backtesting.py | 4 ++-- freqtrade/tests/optimize/test_backtesting.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 53071efaf..3fd96221b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -91,8 +91,8 @@ class Backtesting(object): self.strategy = strategy self.ticker_interval = self.config.get('ticker_interval') self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe - self.populate_buy_trend = strategy.populate_buy_trend - self.populate_sell_trend = strategy.populate_sell_trend + self.advise_buy = strategy.advise_buy + self.advise_sell = strategy.advise_sell @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 02f16be85..0099a3e32 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -775,8 +775,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): def test_backtest_start_multi_strat(default_conf, mocker, caplog): - conf = deepcopy(default_conf) - conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] + default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', new=lambda s, n, i: _load_pair_as_ticks(n, i)) patch_exchange(mocker) @@ -788,7 +787,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table_strategy', gen_strattable_mock) mocker.patch('freqtrade.configuration.open', mocker.mock_open( - read_data=json.dumps(conf) + read_data=json.dumps(default_conf) )) args = [ From 36f91fcdf564ad700534e06e46526b8b0beffb31 Mon Sep 17 00:00:00 2001 From: creslin Date: Wed, 1 Aug 2018 06:03:34 +0000 Subject: [PATCH 17/51] XBT missing as a market symbol for BTC in constants --- freqtrade/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 87e354455..b30add71b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -36,7 +36,7 @@ SUPPORTED_FIAT = [ "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD", - "BTC", "ETH", "XRP", "LTC", "BCH", "USDT" + "BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT" ] # Required json-schema for user specified config @@ -45,7 +45,7 @@ CONF_SCHEMA = { 'properties': { 'max_open_trades': {'type': 'integer', 'minimum': 0}, 'ticker_interval': {'type': 'string', 'enum': list(TICKER_INTERVAL_MINUTES.keys())}, - 'stake_currency': {'type': 'string', 'enum': ['BTC', 'ETH', 'USDT', 'EUR', 'USD']}, + 'stake_currency': {'type': 'string', 'enum': ['BTC', 'XBT', 'ETH', 'USDT', 'EUR', 'USD']}, 'stake_amount': { "type": ["number", "string"], "minimum": 0.0005, From f619cd1d2aae971098d47f03480c396e027e631a Mon Sep 17 00:00:00 2001 From: creslin Date: Thu, 2 Aug 2018 08:45:28 +0000 Subject: [PATCH 18/51] renamed/refactored get_ticker_history to get_candle_history as it does not fetch any ticker data only candles and is causing confusion when developer are talking about candles /tickers incorreclty. OHLCV < candles and Tickers are two seperate datafeeds from the exchange --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/tests/exchange/test_exchange.py | 16 ++++++++-------- freqtrade/tests/test_freqtradebot.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 46fbb3a38..706435017 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -330,7 +330,7 @@ class FreqtradeBot(object): # Pick pair based on buy signals for _pair in whitelist: - thistory = self.exchange.get_ticker_history(_pair, interval) + thistory = self.exchange.get_candle_history(_pair, interval) (buy, sell) = self.strategy.get_signal(_pair, interval, thistory) if buy and not sell: @@ -497,7 +497,7 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - ticker = self.exchange.get_ticker_history(trade.pair, self.strategy.ticker_interval) + ticker = self.exchange.get_candle_history(trade.pair, self.strategy.ticker_interval) (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, ticker) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index d327b97c7..6918e9da1 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -524,7 +524,7 @@ def make_fetch_ohlcv_mock(data): return fetch_ohlcv_mock -def test_get_ticker_history(default_conf, mocker): +def test_get_candle_history(default_conf, mocker): api_mock = MagicMock() tick = [ [ @@ -541,7 +541,7 @@ def test_get_ticker_history(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) # retrieve original ticker - ticks = exchange.get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert ticks[0][0] == 1511686200000 assert ticks[0][1] == 1 assert ticks[0][2] == 2 @@ -563,7 +563,7 @@ def test_get_ticker_history(default_conf, mocker): api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(new_tick)) exchange = get_patched_exchange(mocker, default_conf, api_mock) - ticks = exchange.get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert ticks[0][0] == 1511686210000 assert ticks[0][1] == 6 assert ticks[0][2] == 7 @@ -572,16 +572,16 @@ def test_get_ticker_history(default_conf, mocker): assert ticks[0][5] == 10 ccxt_exceptionhandlers(mocker, default_conf, api_mock, - "get_ticker_history", "fetch_ohlcv", + "get_candle_history", "fetch_ohlcv", pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) with pytest.raises(OperationalException, match=r'Exchange .* does not support.*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.get_ticker_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) + exchange.get_candle_history(pair='ABCD/BTC', tick_interval=default_conf['ticker_interval']) -def test_get_ticker_history_sort(default_conf, mocker): +def test_get_candle_history_sort(default_conf, mocker): api_mock = MagicMock() # GDAX use-case (real data from GDAX) @@ -604,7 +604,7 @@ def test_get_ticker_history_sort(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) # Test the ticker history sort - ticks = exchange.get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert ticks[0][0] == 1527830400000 assert ticks[0][1] == 0.07649 assert ticks[0][2] == 0.07651 @@ -637,7 +637,7 @@ def test_get_ticker_history_sort(default_conf, mocker): api_mock.fetch_ohlcv = MagicMock(side_effect=make_fetch_ohlcv_mock(tick)) exchange = get_patched_exchange(mocker, default_conf, api_mock) # Test the ticker history sort - ticks = exchange.get_ticker_history('ETH/BTC', default_conf['ticker_interval']) + ticks = exchange.get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert ticks[0][0] == 1527827700000 assert ticks[0][1] == 0.07659999 assert ticks[0][2] == 0.0766 diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 69f349107..df73fff3c 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :return: None """ freqtrade.strategy.get_signal = lambda e, s, t: value - freqtrade.exchange.get_ticker_history = lambda p, i: None + freqtrade.exchange.get_candle_history = lambda p, i: None def patch_RPCManager(mocker) -> MagicMock: From a741f1144a43ec7116718cb5e8128b1ee41b8c77 Mon Sep 17 00:00:00 2001 From: creslin Date: Thu, 2 Aug 2018 08:58:04 +0000 Subject: [PATCH 19/51] missing __init__.py --- freqtrade/exchange/__init__.py | 2 +- freqtrade/exchange/exchange_helpers.py | 2 +- freqtrade/optimize/__init__.py | 2 +- freqtrade/optimize/backtesting.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 8 ++++---- freqtrade/tests/optimize/test_optimize.py | 16 ++++++++-------- freqtrade/tests/strategy/test_interface.py | 2 +- scripts/plot_dataframe.py | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 423e38246..0be89aaa5 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -330,7 +330,7 @@ class Exchange(object): return self._cached_ticker[pair] @retrier - def get_ticker_history(self, pair: str, tick_interval: str, + def get_candle_history(self, pair: str, tick_interval: str, since_ms: Optional[int] = None) -> List[Dict]: try: # last item should be in the time interval [now - tick_interval, now] diff --git a/freqtrade/exchange/exchange_helpers.py b/freqtrade/exchange/exchange_helpers.py index 254c16309..46f04328c 100644 --- a/freqtrade/exchange/exchange_helpers.py +++ b/freqtrade/exchange/exchange_helpers.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) def parse_ticker_dataframe(ticker: list) -> DataFrame: """ Analyses the trend for the given ticker history - :param ticker: See exchange.get_ticker_history + :param ticker: See exchange.get_candle_history :return: DataFrame """ cols = ['date', 'open', 'high', 'low', 'close', 'volume'] diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index e806ff2b8..8d5350fe5 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -219,7 +219,7 @@ def download_backtesting_testdata(datadir: str, logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') - new_data = exchange.get_ticker_history(pair=pair, tick_interval=tick_interval, + new_data = exchange.get_candle_history(pair=pair, tick_interval=tick_interval, since_ms=since_ms) data.extend(new_data) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 593af619c..fff658b6f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -283,7 +283,7 @@ class Backtesting(object): if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') for pair in pairs: - data[pair] = self.exchange.get_ticker_history(pair, self.ticker_interval) + data[pair] = self.exchange.get_candle_history(pair, self.ticker_interval) else: logger.info('Using local backtesting data (using whitelist in given config) ...') diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 5d121d27c..fc7b1f043 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -110,7 +110,7 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals return pairdata -# use for mock freqtrade.exchange.get_ticker_history' +# use for mock freqtrade.exchange.get_candle_history' def _load_pair_as_ticks(pair, tickfreq): ticks = optimize.load_data(None, ticker_interval=tickfreq, pairs=[pair]) ticks = trim_dictlist(ticks, -201) @@ -411,7 +411,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history') + mocker.patch('freqtrade.exchange.Exchange.get_candle_history') patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', @@ -446,7 +446,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history') + mocker.patch('freqtrade.exchange.Exchange.get_candle_history') patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', @@ -677,7 +677,7 @@ def test_backtest_record(default_conf, fee, mocker): def test_backtest_start_live(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', new=lambda s, n, i: _load_pair_as_ticks(n, i)) patch_exchange(mocker) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock()) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index eef79bef3..13f65fbf5 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -53,7 +53,7 @@ def _clean_test_file(file: str) -> None: def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-30m.json') _backup_file(file, copy_file=True) optimize.load_data(None, pairs=['UNITTEST/BTC'], ticker_interval='30m') @@ -63,7 +63,7 @@ def test_load_data_30min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-5m.json') _backup_file(file, copy_file=True) @@ -74,7 +74,7 @@ def test_load_data_5min_ticker(ticker_history, mocker, caplog, default_conf) -> def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') _backup_file(file, copy_file=True) optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) @@ -87,7 +87,7 @@ def test_load_data_with_new_pair_1min(ticker_history, mocker, caplog, default_co """ Test load_data() with 1 min ticker """ - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') @@ -118,7 +118,7 @@ def test_testdata_path() -> None: def test_download_pairs(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') @@ -261,7 +261,7 @@ def test_load_cached_data_for_updating(mocker) -> None: def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) mocker.patch('freqtrade.optimize.__init__.download_backtesting_testdata', side_effect=BaseException('File Error')) exchange = get_patched_exchange(mocker, default_conf) @@ -279,7 +279,7 @@ def test_download_pairs_exception(ticker_history, mocker, caplog, default_conf) def test_download_backtesting_testdata(ticker_history, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=ticker_history) exchange = get_patched_exchange(mocker, default_conf) # Download a 1 min ticker file @@ -304,7 +304,7 @@ def test_download_backtesting_testdata2(mocker, default_conf) -> None: [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=tick) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='1m') download_backtesting_testdata(None, exchange, pair="UNITTEST/BTC", tick_interval='3m') diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 2c056870f..ec4ab0fd4 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -88,7 +88,7 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog): def test_get_signal_handles_exceptions(mocker, default_conf): - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', return_value=MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', return_value=MagicMock()) exchange = get_patched_exchange(mocker, default_conf) mocker.patch.object( _STRATEGY, 'analyze_ticker', diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index fbb385a3c..f2f2e0c7f 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -138,7 +138,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: tickers = {} if args.live: logger.info('Downloading pair.') - tickers[pair] = exchange.get_ticker_history(pair, tick_interval) + tickers[pair] = exchange.get_candle_history(pair, tick_interval) else: tickers = optimize.load_data( datadir=_CONF.get("datadir"), From 1f97d0d78b79b6f4ac889a5ba2c8a2004c5b1111 Mon Sep 17 00:00:00 2001 From: creslin Date: Thu, 2 Aug 2018 09:15:02 +0000 Subject: [PATCH 20/51] fix --- freqtrade/tests/test_freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index df73fff3c..89adae6ab 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -544,7 +544,7 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), - get_ticker_history=MagicMock(return_value=20), + get_candle_history=MagicMock(return_value=20), get_balance=MagicMock(return_value=20), get_fee=fee, ) From e282d57a918513209ebdd41e9359e1e782de7dea Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Thu, 2 Aug 2018 12:57:47 +0300 Subject: [PATCH 21/51] fix broken test --- freqtrade/tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index f492384aa..32a5229c0 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -776,7 +776,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): def test_backtest_start_multi_strat(default_conf, mocker, caplog): default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - mocker.patch('freqtrade.exchange.Exchange.get_ticker_history', + mocker.patch('freqtrade.exchange.Exchange.get_candle_history', new=lambda s, n, i: _load_pair_as_ticks(n, i)) patch_exchange(mocker) backtestmock = MagicMock() From 7f4472ad7789b846b37f7107b99baca586f25842 Mon Sep 17 00:00:00 2001 From: creslin Date: Thu, 2 Aug 2018 10:10:44 +0000 Subject: [PATCH 22/51] As requested in issue #1111 A python script to return - all exchanges supported by CCXT - all markets on a exchange Invoked as `python get_market_pairs.py` it will list exchanges Invoked as `python get_market_pairs binance` it will list all markets on binance --- scripts/get_market_pairs.py | 93 +++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 scripts/get_market_pairs.py diff --git a/scripts/get_market_pairs.py b/scripts/get_market_pairs.py new file mode 100644 index 000000000..6ee6464d3 --- /dev/null +++ b/scripts/get_market_pairs.py @@ -0,0 +1,93 @@ +import os +import sys + +root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append(root + '/python') + +import ccxt # noqa: E402 + + +def style(s, style): + return style + s + '\033[0m' + + +def green(s): + return style(s, '\033[92m') + + +def blue(s): + return style(s, '\033[94m') + + +def yellow(s): + return style(s, '\033[93m') + + +def red(s): + return style(s, '\033[91m') + + +def pink(s): + return style(s, '\033[95m') + + +def bold(s): + return style(s, '\033[1m') + + +def underline(s): + return style(s, '\033[4m') + + +def dump(*args): + print(' '.join([str(arg) for arg in args])) + + +def print_supported_exchanges(): + dump('Supported exchanges:', green(', '.join(ccxt.exchanges))) + + +try: + + id = sys.argv[1] # get exchange id from command line arguments + + + # check if the exchange is supported by ccxt + exchange_found = id in ccxt.exchanges + + if exchange_found: + dump('Instantiating', green(id), 'exchange') + + # instantiate the exchange by id + exchange = getattr(ccxt, id)({ + # 'proxy':'https://cors-anywhere.herokuapp.com/', + }) + + # load all markets from the exchange + markets = exchange.load_markets() + + # output a list of all market symbols + dump(green(id), 'has', len(exchange.symbols), 'symbols:', exchange.symbols) + + tuples = list(ccxt.Exchange.keysort(markets).items()) + + # debug + for (k, v) in tuples: + print(v) + + # output a table of all markets + dump(pink('{:<15} {:<15} {:<15} {:<15}'.format('id', 'symbol', 'base', 'quote'))) + + for (k, v) in tuples: + dump('{:<15} {:<15} {:<15} {:<15}'.format(v['id'], v['symbol'], v['base'], v['quote'])) + + else: + + dump('Exchange ' + red(id) + ' not found') + print_supported_exchanges() + +except Exception as e: + dump('[' + type(e).__name__ + ']', str(e)) + dump("Usage: python " + sys.argv[0], green('id')) + print_supported_exchanges() + From 0fc4a7910d01f79491d97b1a37d52cb4cd24c72e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Aug 2018 20:15:18 +0200 Subject: [PATCH 23/51] Add note to readme for binance users --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index da691230f..7b6b4996b 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ hesitate to read the source code and understand the mechanism of this bot. ## Exchange marketplaces supported - [X] [Bittrex](https://bittrex.com/) -- [X] [Binance](https://www.binance.com/) +- [X] [Binance](https://www.binance.com/) ([*Note for binance users](#a-note-on-binance)) - [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ## Features @@ -152,6 +152,13 @@ The project is currently setup in two main branches: - `develop` - This branch has often new features, but might also cause breaking changes. - `master` - This branch contains the latest stable release. The bot 'should' be stable on this branch, and is generally well tested. +- `feat/*` - This are feature branches, which are beeing worked on heavily. Please don't use these unless you want to test a specific feature. + + +## A note on Binance + +For Binance, please add `"BNB/"` to your blacklist to avoid issues. +Accounts having BNB accounts use this to pay for fees - if your first trade happens to be on `BNB`, further trades will consume this position and make the initial BNB order unsellable as the expected amount is not there anymore. ## Support From 00b81e3f0df781c2e718b94b23dff3e467a7e4dd Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Aug 2018 11:45:28 +0200 Subject: [PATCH 24/51] fix readme.md spelling --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b6b4996b..02b870209 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ The project is currently setup in two main branches: - `develop` - This branch has often new features, but might also cause breaking changes. - `master` - This branch contains the latest stable release. The bot 'should' be stable on this branch, and is generally well tested. -- `feat/*` - This are feature branches, which are beeing worked on heavily. Please don't use these unless you want to test a specific feature. +- `feat/*` - These are feature branches, which are beeing worked on heavily. Please don't use these unless you want to test a specific feature. ## A note on Binance From 145008421f9fd62e964366873239b0d83635b874 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 2 Aug 2018 14:26:07 +0200 Subject: [PATCH 25/51] Update ccxt from 1.17.60 to 1.17.63 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5ff5d3694..ff6457a8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.60 +ccxt==1.17.63 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 85c73ea8507d012353bb9744b4371d978bf07af6 Mon Sep 17 00:00:00 2001 From: Gert Date: Thu, 2 Aug 2018 16:39:13 -0700 Subject: [PATCH 26/51] added index --- freqtrade/persistence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 8fb01d074..c21b902bc 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -157,8 +157,8 @@ class Trade(_DECL_BASE): id = Column(Integer, primary_key=True) exchange = Column(String, nullable=False) - pair = Column(String, nullable=False) - is_open = Column(Boolean, nullable=False, default=True) + pair = Column(String, nullable=False,index=True) + is_open = Column(Boolean, nullable=False, default=True, index=True) fee_open = Column(Float, nullable=False, default=0.0) fee_close = Column(Float, nullable=False, default=0.0) open_rate = Column(Float) From 2cfa3b7607874879584484c7c99d47c969517fb5 Mon Sep 17 00:00:00 2001 From: Gert Wohlgemuth Date: Thu, 2 Aug 2018 17:08:14 -0700 Subject: [PATCH 27/51] updated dockerfile and requirements --- Dockerfile | 7 ++++++- requirements.txt | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 309763d2a..10cd14bfe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.6.6-slim-stretch # Install TA-lib -RUN apt-get update && apt-get -y install curl build-essential && apt-get clean +RUN apt-get update && apt-get -y install curl build-essential git && apt-get clean RUN curl -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz | \ tar xzvf - && \ cd ta-lib && \ @@ -13,6 +13,11 @@ ENV LD_LIBRARY_PATH /usr/local/lib RUN mkdir /freqtrade WORKDIR /freqtrade +# Update PIP +RUN python -m pip install --upgrade pip +RUN pip install future +RUN pip install numpy + # Install dependencies COPY requirements.txt /freqtrade/ RUN pip install numpy \ diff --git a/requirements.txt b/requirements.txt index ff6457a8c..183d79cdf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,13 @@ scikit-optimize==0.5.2 # Required for plotting data #plotly==3.0.0 + +# Required for plotting data +plotly==3.0.0 + +# find first, C search in arrays +py_find_1st==1.1.1 + +#Load ticker files 30% faster +ujson==1.35 +git+git://github.com/berlinguyinca/technical.git@master From 3037d85529fc0504506a902a46fb27ca2ae20091 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 3 Aug 2018 14:26:06 +0200 Subject: [PATCH 28/51] Update ccxt from 1.17.63 to 1.17.66 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ff6457a8c..0c523ddec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.63 +ccxt==1.17.66 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From b963b95ee9909d2c03f5ef244c49ac6bc6d78130 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 3 Aug 2018 14:26:07 +0200 Subject: [PATCH 29/51] Update pytest from 3.7.0 to 3.7.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0c523ddec..8670b4074 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.0 TA-Lib==0.4.17 -pytest==3.7.0 +pytest==3.7.1 pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 From 721341e4128a422a33ef6a059db4e7e97a164a9a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 4 Aug 2018 14:26:05 +0200 Subject: [PATCH 30/51] Update ccxt from 1.17.66 to 1.17.73 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8670b4074..221bdf968 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.66 +ccxt==1.17.73 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From ea506b05c67c4da1b66e328bdf5d8b79bf33ed4a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 4 Aug 2018 20:22:16 +0200 Subject: [PATCH 31/51] Add test for failing database migration --- freqtrade/tests/test_persistence.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 26932136a..e52500071 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -404,6 +404,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): Test Database migration (starting with new pairformat) """ amount = 103.223 + # Always create all columns apart from the last! create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( id INTEGER NOT NULL, exchange VARCHAR NOT NULL, @@ -418,14 +419,21 @@ def test_migrate_new(mocker, default_conf, fee, caplog): open_date DATETIME NOT NULL, close_date DATETIME, open_order_id VARCHAR, + stop_loss FLOAT, + initial_stop_loss FLOAT, + max_rate FLOAT, + sell_reason VARCHAR, + strategy VARCHAR, PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, - open_rate, stake_amount, amount, open_date) + open_rate, stake_amount, amount, open_date, + stop_loss, initial_stop_loss, max_rate) VALUES ('binance', 'ETC/BTC', 1, {fee}, 0.00258580, {stake}, {amount}, - '2019-11-28 12:44:24.000000') + '2019-11-28 12:44:24.000000', + 0.0, 0.0, 0.0) """.format(fee=fee.return_value, stake=default_conf.get("stake_amount"), amount=amount From d73d0a5253016b65fadb97578a5eb5b1c80180c8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 4 Aug 2018 20:22:45 +0200 Subject: [PATCH 32/51] Fix database migration --- freqtrade/persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 8fb01d074..6eaa5008a 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -82,7 +82,7 @@ def check_migrate(engine) -> None: logger.info(f'trying {table_back_name}') # Check for latest column - if not has_column(cols, 'max_rate'): + if not has_column(cols, 'ticker_interval'): fee_open = get_column_def(cols, 'fee_open', 'fee') fee_close = get_column_def(cols, 'fee_close', 'fee') open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') From be9436b2a6d7e50bd61c90cc51a832e0cd13ceb8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 5 Aug 2018 14:26:07 +0200 Subject: [PATCH 33/51] Update ccxt from 1.17.73 to 1.17.78 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 221bdf968..3c2b10847 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.73 +ccxt==1.17.78 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From ba4de4137e033319ed34ce8ffca155f56b100480 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 5 Aug 2018 14:26:08 +0200 Subject: [PATCH 34/51] Update pandas from 0.23.3 to 0.23.4 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3c2b10847..edeb07527 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cachetools==2.1.0 requests==2.19.1 urllib3==1.22 wrapt==1.10.11 -pandas==0.23.3 +pandas==0.23.4 scikit-learn==0.19.2 scipy==1.1.0 jsonschema==2.6.0 From 0b825e96aac2bf5e93606309ccc12a209cdf6582 Mon Sep 17 00:00:00 2001 From: Axel Cherubin Date: Sun, 5 Aug 2018 16:08:49 -0400 Subject: [PATCH 35/51] fix talib bug on bollinger bands and other indicators when working on small assets, rise talib prescision and add test associated --- Dockerfile | 1 + freqtrade/tests/test_talib.py | 15 +++++++++++++++ install_ta-lib.sh | 8 ++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 freqtrade/tests/test_talib.py diff --git a/Dockerfile b/Dockerfile index 309763d2a..e959b9296 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ RUN apt-get update && apt-get -y install curl build-essential && apt-get clean RUN curl -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz | \ tar xzvf - && \ cd ta-lib && \ + sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ ./configure && make && make install && \ cd .. && rm -rf ta-lib ENV LD_LIBRARY_PATH /usr/local/lib diff --git a/freqtrade/tests/test_talib.py b/freqtrade/tests/test_talib.py new file mode 100644 index 000000000..f5e51c553 --- /dev/null +++ b/freqtrade/tests/test_talib.py @@ -0,0 +1,15 @@ + + +import talib.abstract as ta +import pandas as pd + +def test_talib_bollingerbands_near_zero_values(): + inputs = pd.DataFrame([ + {'close': 0.00000010}, + {'close': 0.00000011}, + {'close': 0.00000012}, + {'close': 0.00000013}, + {'close': 0.00000014} + ]) + bollinger = ta.BBANDS(inputs, matype=0, timeperiod=2) + assert (bollinger['upperband'][3] != bollinger['middleband'][3]) \ No newline at end of file diff --git a/install_ta-lib.sh b/install_ta-lib.sh index 21e69cbba..d5d7cf03e 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,7 +1,11 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib && ./configure && make && sudo make install && cd .. + cd ta-lib && \ + sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ + ./configure && make && sudo make install && cd .. else echo "TA-lib already installed, skipping download and build." - cd ta-lib && sudo make install && cd .. + cd ta-lib && \ + sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ + sudo make install && cd .. fi From a5554604e0e3a9a01582d5221f13e754766d7e87 Mon Sep 17 00:00:00 2001 From: Axel Cherubin Date: Sun, 5 Aug 2018 16:59:18 -0400 Subject: [PATCH 36/51] add sed command in doc, fix travis error --- docs/installation.md | 1 + install_ta-lib.sh | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 7a7719fc0..4de05c121 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -267,6 +267,7 @@ Official webpage: https://mrjbq7.github.io/ta-lib/install.html wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz tar xvzf ta-lib-0.4.0-src.tar.gz cd ta-lib +sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h ./configure --prefix=/usr make make install diff --git a/install_ta-lib.sh b/install_ta-lib.sh index d5d7cf03e..1639bd3a2 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -1,11 +1,7 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib && \ - sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ - ./configure && make && sudo make install && cd .. + cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. else echo "TA-lib already installed, skipping download and build." - cd ta-lib && \ - sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && \ - sudo make install && cd .. + cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && sudo make install && cd .. fi From 848ecb91bbb537e834cc38221d2360fd4a0118a0 Mon Sep 17 00:00:00 2001 From: Axel Cherubin Date: Sun, 5 Aug 2018 17:28:53 -0400 Subject: [PATCH 37/51] remove unnecessary seb command --- install_ta-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_ta-lib.sh b/install_ta-lib.sh index 1639bd3a2..18e7b8bbb 100755 --- a/install_ta-lib.sh +++ b/install_ta-lib.sh @@ -3,5 +3,5 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && ./configure && make && sudo make install && cd .. else echo "TA-lib already installed, skipping download and build." - cd ta-lib && sed -i "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h && sudo make install && cd .. + cd ta-lib && sudo make install && cd .. fi From 65f7b75c343693ed560a15addaac6413865fd865 Mon Sep 17 00:00:00 2001 From: Axel Cherubin Date: Sun, 5 Aug 2018 17:52:06 -0400 Subject: [PATCH 38/51] fix flake8 issue --- freqtrade/tests/test_talib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_talib.py b/freqtrade/tests/test_talib.py index f5e51c553..093c3023c 100644 --- a/freqtrade/tests/test_talib.py +++ b/freqtrade/tests/test_talib.py @@ -3,6 +3,7 @@ import talib.abstract as ta import pandas as pd + def test_talib_bollingerbands_near_zero_values(): inputs = pd.DataFrame([ {'close': 0.00000010}, @@ -12,4 +13,4 @@ def test_talib_bollingerbands_near_zero_values(): {'close': 0.00000014} ]) bollinger = ta.BBANDS(inputs, matype=0, timeperiod=2) - assert (bollinger['upperband'][3] != bollinger['middleband'][3]) \ No newline at end of file + assert (bollinger['upperband'][3] != bollinger['middleband'][3]) From bc62f626c529ed7478f7876bf52b7c6dd2fb42a3 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 6 Aug 2018 14:26:06 +0200 Subject: [PATCH 39/51] Update ccxt from 1.17.78 to 1.17.81 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index edeb07527..f3135f9bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.78 +ccxt==1.17.81 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 131d268721f7a9499961db619d337261ee7b4f62 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Aug 2018 19:15:30 +0200 Subject: [PATCH 40/51] Fix failing tests when metadata in `analyze_ticker` is actually used --- freqtrade/tests/test_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_dataframe.py b/freqtrade/tests/test_dataframe.py index ce144e118..dc030d630 100644 --- a/freqtrade/tests/test_dataframe.py +++ b/freqtrade/tests/test_dataframe.py @@ -14,7 +14,7 @@ def load_dataframe_pair(pairs, strategy): assert isinstance(pairs[0], str) dataframe = ld[pairs[0]] - dataframe = strategy.analyze_ticker(dataframe, pairs[0]) + dataframe = strategy.analyze_ticker(dataframe, {'pair': pairs[0]}) return dataframe From 3d94720be98953f658f0c96d867d055a0c6d5f91 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 7 Aug 2018 14:26:07 +0200 Subject: [PATCH 41/51] Update ccxt from 1.17.81 to 1.17.84 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f3135f9bb..2db78bd2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.81 +ccxt==1.17.84 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 4d03fc213f51acbe5a23a5f7e13e94c5ad02b428 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 8 Aug 2018 14:26:07 +0200 Subject: [PATCH 42/51] Update ccxt from 1.17.84 to 1.17.86 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2db78bd2c..82c739a70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.84 +ccxt==1.17.86 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 1bcd4333fc3ffef27e978f33c3d8b47e554414f4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 9 Aug 2018 14:26:06 +0200 Subject: [PATCH 43/51] Update ccxt from 1.17.86 to 1.17.94 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 82c739a70..91ecf71c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.86 +ccxt==1.17.94 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 5bec389e853ec6ab9c6fd48a0b2866af4e1fd069 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 11 Aug 2018 14:26:06 +0200 Subject: [PATCH 44/51] Update ccxt from 1.17.94 to 1.17.106 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 91ecf71c9..d3ff4e6d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.94 +ccxt==1.17.106 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 5f8ec82319f63630db3f58f15b0ab6d6c3c17284 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Aug 2018 09:18:30 +0200 Subject: [PATCH 45/51] Revert "updated dockerfile and requirements" This reverts commit 2cfa3b7607874879584484c7c99d47c969517fb5. --- Dockerfile | 7 +------ requirements.txt | 10 ---------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 10cd14bfe..309763d2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.6.6-slim-stretch # Install TA-lib -RUN apt-get update && apt-get -y install curl build-essential git && apt-get clean +RUN apt-get update && apt-get -y install curl build-essential && apt-get clean RUN curl -L http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz | \ tar xzvf - && \ cd ta-lib && \ @@ -13,11 +13,6 @@ ENV LD_LIBRARY_PATH /usr/local/lib RUN mkdir /freqtrade WORKDIR /freqtrade -# Update PIP -RUN python -m pip install --upgrade pip -RUN pip install future -RUN pip install numpy - # Install dependencies COPY requirements.txt /freqtrade/ RUN pip install numpy \ diff --git a/requirements.txt b/requirements.txt index 183d79cdf..ff6457a8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,13 +23,3 @@ scikit-optimize==0.5.2 # Required for plotting data #plotly==3.0.0 - -# Required for plotting data -plotly==3.0.0 - -# find first, C search in arrays -py_find_1st==1.1.1 - -#Load ticker files 30% faster -ujson==1.35 -git+git://github.com/berlinguyinca/technical.git@master From ffa47151ee50ece9b00dece77e6fe3f0e6edfabf Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Aug 2018 09:30:12 +0200 Subject: [PATCH 46/51] Flake8 fix --- freqtrade/persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index c21b902bc..a169bc042 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -157,7 +157,7 @@ class Trade(_DECL_BASE): id = Column(Integer, primary_key=True) exchange = Column(String, nullable=False) - pair = Column(String, nullable=False,index=True) + pair = Column(String, nullable=False, index=True) is_open = Column(Boolean, nullable=False, default=True, index=True) fee_open = Column(Float, nullable=False, default=0.0) fee_close = Column(Float, nullable=False, default=0.0) From 2e7837976da309dfcdc7d85186a5268766e21ae2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 12 Aug 2018 14:26:06 +0200 Subject: [PATCH 47/51] Update ccxt from 1.17.106 to 1.17.113 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d3ff4e6d7..c1bd768c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.106 +ccxt==1.17.113 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From eca8682528d525885149693a1b6a1946ff08838a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 13 Aug 2018 14:26:06 +0200 Subject: [PATCH 48/51] Update ccxt from 1.17.113 to 1.17.118 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c1bd768c5..d373f8c73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.113 +ccxt==1.17.118 SQLAlchemy==1.2.10 python-telegram-bot==10.1.0 arrow==0.12.1 From 2602cbe6832c200174f0051fcd6337a383237b02 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 13:21:15 +0200 Subject: [PATCH 49/51] add async method --- freqtrade/arguments.py | 16 +- freqtrade/optimize/backslapping.py | 112 ++++---- freqtrade/optimize/backtesting.py | 424 +++-------------------------- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/optimize/optimize.py | 322 ++++++++++++++++++++++ 5 files changed, 428 insertions(+), 448 deletions(-) create mode 100644 freqtrade/optimize/optimize.py diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 1f6de2052..734f9b4f8 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -161,14 +161,6 @@ class Arguments(object): dest='exportfilename', metavar='PATH', ) - parser.add_argument( - '--backslap', - help="Utilize the Backslapping approach instead of the default Backtesting. This should provide more " - "accurate results, unless you are utilizing Min/Max function in your strategy.", - required=False, - dest='backslap', - action='store_true' - ) @staticmethod def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: @@ -236,7 +228,7 @@ class Arguments(object): Builds and attaches all subcommands :return: None """ - from freqtrade.optimize import backtesting, hyperopt + from freqtrade.optimize import backtesting, backslapping, hyperopt subparsers = self.parser.add_subparsers(dest='subparser') @@ -246,6 +238,12 @@ class Arguments(object): self.optimizer_shared_options(backtesting_cmd) self.backtesting_options(backtesting_cmd) + # Add backslapping subcommand + backslapping_cmd = subparsers.add_parser('backslapping', help='backslapping module') + backslapping_cmd.set_defaults(func=backslapping.start) + self.optimizer_shared_options(backslapping_cmd) + self.backtesting_options(backslapping_cmd) + # Add hyperopt subcommand hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module') hyperopt_cmd.set_defaults(func=hyperopt.start) diff --git a/freqtrade/optimize/backslapping.py b/freqtrade/optimize/backslapping.py index b16515942..6c8e1ae86 100644 --- a/freqtrade/optimize/backslapping.py +++ b/freqtrade/optimize/backslapping.py @@ -1,48 +1,35 @@ import timeit +from argparse import Namespace +import logging from typing import Dict, Any from pandas import DataFrame from freqtrade.exchange import Exchange +from freqtrade.optimize.optimize import IOptimize, BacktestResult, setup_configuration from freqtrade.strategy import IStrategy from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import StrategyResolver +logger = logging.getLogger(__name__) -class Backslapping: + +class Backslapping(IOptimize): """ provides a quick way to evaluate strategies over a longer term of time """ - def __init__(self, config: Dict[str, Any], exchange = None) -> None: + def __init__(self, config: Dict[str, Any]) -> None: """ constructor """ - - self.config = config - self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.ticker_interval = self.strategy.ticker_interval - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.populate_buy_trend = self.strategy.populate_buy_trend - self.populate_sell_trend = self.strategy.populate_sell_trend - - ### - # - ### - if exchange is None: - self.config['exchange']['secret'] = '' - self.config['exchange']['password'] = '' - self.config['exchange']['uid'] = '' - self.config['dry_run'] = True - self.exchange = Exchange(self.config) - else: - self.exchange = exchange + super().__init__(config) self.fee = self.exchange.get_fee() self.stop_loss_value = self.strategy.stoploss - #### backslap config + # backslap config ''' Numpy arrays are used for 100x speed up We requires setting Int values for @@ -81,7 +68,7 @@ class Backslapping: def f(self, st): return (timeit.default_timer() - st) - def run(self,args): + def run(self, args): headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] processed = args['processed'] @@ -96,8 +83,8 @@ class Backslapping: if self.debug_timing: # Start timer fl = self.s() - ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data))[headers].copy() + ticker_data = self.advise_sell(self.advise_buy(pair_data, {'pair': pair}), + {'pair': pair})[headers].copy() if self.debug_timing: # print time taken flt = self.f(fl) @@ -132,7 +119,7 @@ class Backslapping: bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair) else: - from freqtrade.optimize.backtesting import BacktestResult + bslap_results_df = [] bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) @@ -221,13 +208,13 @@ class Backslapping: """ The purpose of this def is to return the next "buy" = 1 after t_exit_ind. - - This function will also check is the stop limit for the pair has been reached. + + This function will also check is the stop limit for the pair has been reached. if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. t_exit_ind is the index the last trade exited on or 0 if first time around this loop. - + stop_stops i """ debug = self.debug @@ -379,7 +366,7 @@ class Backslapping: a) Find first buy index b) Discover first stop and sell hit after buy index c) Chose first instance as trade exit - + Phase 2 2) Manage dynamic Stop and ROI Exit a) Create trade slice from 1 @@ -392,14 +379,14 @@ class Backslapping: ''' 0 - Find next buy entry Finds index for first (buy = 1) flag - + Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" Required: t_exit_ind - Either 0, first loop. Or The index we last exited on - Requires: np_buy_arr_len - length of pair array. - Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair Requires: stop_stop_counts - count of stops hit in the pair Provides: The next "buy" index after t_exit_ind - + If -1 is returned no buy has been found in remainder of array, skip to exit loop ''' t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) @@ -416,19 +403,19 @@ class Backslapping: """ 1 - Create views to search within for our open trade - + The views are our search space for the next Stop or Sell Numpy view is employed as: 1,000 faster than pandas searches Pandas cannot assure it will always return a view, it may make a slow copy. - + The view contains columns: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - + Requires: np_bslap is our numpy array of the ticker DataFrame Requires: t_open_ind is the index row with the buy. Provides: np_t_open_v View of array after buy. - Provides: np_t_open_v_stop View of array after buy +1 + Provides: np_t_open_v_stop View of array after buy +1 (Stop will search in here to prevent stopping in the past) """ np_t_open_v = np_bslap[t_open_ind:] @@ -446,13 +433,13 @@ class Backslapping: ''' 2 - Calculate our stop-loss price - + As stop is based on buy price of our trade - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. This is as we only see the CLOSE after it has happened. The back test assumption is we have bought at first available price, the OPEN - + Requires: np_bslap - is our numpy array of the ticker DataFrame Requires: t_open_ind - is the index row with the first buy. Requires: p_stop - is the stop rate, ie. 0.99 is -1% @@ -469,9 +456,9 @@ class Backslapping: ''' 3 - Find candle STO is under Stop-Loss After Trade opened. - + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - + Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop @@ -501,9 +488,9 @@ class Backslapping: ''' 4 - Find first sell index after trade open - + First index in the view np_t_open_v where ['sell'] = 1 - + Requires: np_t_open_v - view of ticker_data from buy onwards Requires: no_sell - integer '3', the buy column in the array Provides: np_t_sell_ind index of view where first sell=1 after buy @@ -528,13 +515,13 @@ class Backslapping: ''' 5 - Determine which was hit first a stop or sell To then use as exit index price-field (sell on buy, stop on stop) - + STOP takes priority over SELL as would be 'in candle' from tick data Sell would use Open from Next candle. So in a draw Stop would be hit first on ticker data in live - + Validity of when types of trades may be executed can be summarised as: - + Tick View index index Buy Sell open low close high Stop price open 2am 94 -1 0 0 ----- ------ ------ ----- ----- @@ -542,25 +529,25 @@ class Backslapping: open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out open 5am 97 2 0 0 Exit ------ ------- ----- ----- open 6am 98 3 0 0 ----- ------ ------- ----- ----- - + -1 means not found till end of view i.e no valid Stop found. Exclude from match. Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. - + Buys and sells are triggered at candle close Both will open their postions at the open of the next candle. i/e + 1 index - + Stop and buy Indexes are on the view. To map to the ticker dataframe the t_open_ind index should be summed. - + np_t_stop_ind: Stop Found index in view t_exit_ind : Sell found in view t_open_ind : Where view was started on ticker_data - + TODO: fix this frig for logic test,, case/switch/dictionary would be better... more so when later testing many options, dynamic stop / roi etc cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) - + ''' if debug: print("\n(5) numpy debug\nStop or Sell Logic Processing") @@ -730,7 +717,7 @@ class Backslapping: if t_exit_last >= t_exit_ind or t_exit_last == -1: """ Break loop and go on to next pair. - + When last trade exit equals index of last exit, there is no opportunity to close any more trades. """ @@ -763,7 +750,7 @@ class Backslapping: bslap_result["open_rate"] = round(np_trade_enter_price, 15) bslap_result["close_rate"] = round(np_trade_exit_price, 15) bslap_result["exit_type"] = t_exit_type - bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care + bslap_result["sell_reason"] = t_exit_type # duplicated, but I don't care # append the dict to the list and print list bslap_pair_results.append(bslap_result) @@ -787,3 +774,18 @@ class Backslapping: # Send back List of trade dicts return bslap_pair_results + + +def start(args: Namespace) -> None: + """ + Start Backtesting script + :param args: Cli args from Arguments() + :return: None + """ + # Initialize configuration + config = setup_configuration(args) + logger.info('Starting freqtrade in Backtesting mode') + + # Initialize backtesting object + backslapping = Backslapping(config) + backslapping.start() diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d6de6cb0a..aa841adac 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -4,51 +4,21 @@ This module contains the backtesting logic """ import logging -import operator from argparse import Namespace -from datetime import datetime, timedelta -from typing import Any, Dict, List, NamedTuple, Optional, Tuple +from typing import Any, Dict, List, Optional -import arrow -from pandas import DataFrame, to_datetime -from tabulate import tabulate +from pandas import DataFrame import freqtrade.optimize as optimize -from freqtrade import DependencyException, constants +from freqtrade.optimize.optimize import IOptimize, BacktestResult, setup_configuration from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration -from freqtrade.exchange import Exchange -from freqtrade.misc import file_dump_json -from freqtrade.optimize.backslapping import Backslapping from freqtrade.persistence import Trade from freqtrade.strategy.interface import SellType -from freqtrade.strategy.resolver import IStrategy, StrategyResolver -from collections import OrderedDict -import timeit -from time import sleep logger = logging.getLogger(__name__) -class BacktestResult(NamedTuple): - """ - NamedTuple Defining BacktestResults inputs. - """ - pair: str - profit_percent: float - profit_abs: float - open_time: datetime - close_time: datetime - open_index: int - close_index: int - trade_duration: float - open_at_end: bool - open_rate: float - close_rate: float - sell_reason: SellType - - -class Backtesting(object): +class Backtesting(IOptimize): """ Backtesting class, this class contains all the logic to run a backtest @@ -58,139 +28,7 @@ class Backtesting(object): """ def __init__(self, config: Dict[str, Any]) -> None: - self.config = config - self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.ticker_interval = self.strategy.ticker_interval - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.advise_buy = self.strategy.advise_buy - self.advise_sell = self.strategy.advise_sell - - # Reset keys for backtesting - self.config['exchange']['key'] = '' - self.config['exchange']['secret'] = '' - self.config['exchange']['password'] = '' - self.config['exchange']['uid'] = '' - self.config['dry_run'] = True - self.exchange = Exchange(self.config) - self.fee = self.exchange.get_fee() - - self.stop_loss_value = self.strategy.stoploss - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - self.np_buy: int = 0 - self.np_open: int = 1 - self.np_close: int = 2 - self.np_sell: int = 3 - self.np_high: int = 4 - self.np_low: int = 5 - self.np_stop: int = 6 - self.np_bto: int = self.np_close # buys_triggered_on - should be close - self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - - if 'backslap' in config: - self.use_backslap = config['backslap'] # Enable backslap - if false Orginal code is executed. - else: - self.use_backslap = False - - logger.info("using backslap: {}".format(self.use_backslap)) - - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt - - self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit - - self.backslap = Backslapping(config) - - @staticmethod - def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: - """ - Get the maximum timeframe for the given backtest data - :param data: dictionary with preprocessed backtesting data - :return: tuple containing min_date, max_date - """ - timeframe = [ - (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) - for frame in data.values() - ] - return min(timeframe, key=operator.itemgetter(0))[0], \ - max(timeframe, key=operator.itemgetter(1))[1] - - def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: - """ - Generates and returns a text table for the given backtest data and the results dataframe - :return: pretty printed table with tabulate as str - """ - stake_currency = str(self.config.get('stake_currency')) - - floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f') - tabular_data = [] - headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', - 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] - for pair in data: - result = results[results.pair == pair] - tabular_data.append([ - pair, - len(result.index), - result.profit_percent.mean() * 100.0, - result.profit_percent.sum() * 100.0, - result.profit_abs.sum(), - str(timedelta( - minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00', - len(result[result.profit_abs > 0]), - len(result[result.profit_abs < 0]) - ]) - - # Append Total - tabular_data.append([ - 'TOTAL', - len(results.index), - results.profit_percent.mean() * 100.0, - results.profit_percent.sum() * 100.0, - results.profit_abs.sum(), - str(timedelta( - minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', - len(results[results.profit_abs > 0]), - len(results[results.profit_abs < 0]) - ]) - return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") - - def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str: - """ - Generate small table outlining Backtest results - """ - - tabular_data = [] - headers = ['Sell Reason', 'Count'] - for reason, count in results['sell_reason'].value_counts().iteritems(): - tabular_data.append([reason.value, count]) - return tabulate(tabular_data, headers=headers, tablefmt="pipe") - - def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: - - records = [(t.pair, t.profit_percent, t.open_time.timestamp(), - t.close_time.timestamp(), t.open_index - 1, t.trade_duration, - t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value) - for index, t in results.iterrows()] - - if records: - logger.info('Dumping backtest results to %s', recordfilename) - file_dump_json(recordfilename, records) + super().__init__(config) def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, @@ -217,13 +55,14 @@ class Backtesting(object): sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, sell_row.sell) if sell.sell_flag: + return BacktestResult(pair=pair, profit_percent=trade.calc_profit_percent(rate=sell_row.open), profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, trade_duration=int(( - sell_row.date - buy_row.date).total_seconds() // 60), + sell_row.date - buy_row.date).total_seconds() // 60), open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=False, @@ -240,7 +79,7 @@ class Backtesting(object): open_time=buy_row.date, close_time=sell_row.date, trade_duration=int(( - sell_row.date - buy_row.date).total_seconds() // 60), + sell_row.date - buy_row.date).total_seconds() // 60), open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=True, @@ -253,14 +92,7 @@ class Backtesting(object): return btr return None - def s(self): - st = timeit.default_timer() - return st - - def f(self, st): - return (timeit.default_timer() - st) - - def backtest(self, args: Dict) -> DataFrame: + def run(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -275,50 +107,32 @@ class Backtesting(object): position_stacking: do we allow position stacking? (default: False) :return: DataFrame """ + headers = ['date', 'buy', 'open', 'close', 'sell'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + position_stacking = args.get('position_stacking', False) + trades = [] + trade_count_lock: Dict = {} + for pair, pair_data in processed.items(): + pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - use_backslap = self.use_backslap - debug_timing = self.debug_timing_main_loop - - if use_backslap: # Use Back Slap code - return self.backslap.run(args) - else: # use Original Back test code - ########################## Original BT loop - - headers = ['date', 'buy', 'open', 'close', 'sell'] - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - position_stacking = args.get('position_stacking', False) - trades = [] - trade_count_lock: Dict = {} - - for pair, pair_data in processed.items(): - if debug_timing: # Start timer - fl = self.s() - - pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - - ticker_data = self.advise_sell( + ticker_data = self.advise_sell( self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() - # to avoid using data from future, we buy/sell with signal from previous candle - ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) - ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + # to avoid using data from future, we buy/sell with signal from previous candle + ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) + ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) - ticker_data.drop(ticker_data.head(1).index, inplace=True) + ticker_data.drop(ticker_data.head(1).index, inplace=True) - if debug_timing: # print time taken - flt = self.f(fl) - # print("populate_buy_trend:", pair, round(flt, 10)) - st = self.s() + # Convert from Pandas to list for performance reasons + # (Looping Pandas is slow.) + ticker = [x for x in ticker_data.itertuples()] - # Convert from Pandas to list for performance reasons - # (Looping Pandas is slow.) - ticker = [x for x in ticker_data.itertuples()] - - lock_pair_until = None - for index, row in enumerate(ticker): - if row.buy == 0 or row.sell == 1: - continue # skip rows where no buy signal or that would immediately sell off + lock_pair_until = None + for index, row in enumerate(ticker): + if row.buy == 0 or row.sell == 1: + continue # skip rows where no buy signal or that would immediately sell off if not position_stacking: if lock_pair_until is not None and row.date <= lock_pair_until: @@ -328,178 +142,22 @@ class Backtesting(object): if not trade_count_lock.get(row.date, 0) < max_open_trades: continue - trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 - trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], - trade_count_lock, args) + trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + trade_count_lock, args) - if trade_entry: - lock_pair_until = trade_entry.close_time - trades.append(trade_entry) - else: - # Set lock_pair_until to end of testing period if trade could not be closed - # This happens only if the buy-signal was with the last candle - lock_pair_until = ticker_data.iloc[-1].date + if trade_entry: + lock_pair_until = trade_entry.close_time + trades.append(trade_entry) + else: + # Set lock_pair_until to end of testing period if trade could not be closed + # This happens only if the buy-signal was with the last candle + lock_pair_until = ticker_data.iloc[-1].date - if debug_timing: # print time taken - tt = self.f(st) - print("Time to BackTest :", pair, round(tt, 10)) - print("-----------------------") + return DataFrame.from_records(trades, columns=BacktestResult._fields) - return DataFrame.from_records(trades, columns=BacktestResult._fields) - ####################### Original BT loop end - - def start(self) -> None: - """ - Run a backtesting end-to-end - :return: None - """ - data = {} - pairs = self.config['exchange']['pair_whitelist'] - logger.info('Using stake_currency: %s ...', self.config['stake_currency']) - logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - - if self.config.get('live'): - logger.info('Downloading data for all pairs in whitelist ...') - for pair in pairs: - data[pair] = self.exchange.get_ticker_history(pair, self.ticker_interval) - else: - logger.info('Using local backtesting data (using whitelist in given config) ...') - - timerange = Arguments.parse_timerange(None if self.config.get( - 'timerange') is None else str(self.config.get('timerange'))) - - data = optimize.load_data( - self.config['datadir'], - pairs=pairs, - ticker_interval=self.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - exchange=self.exchange, - timerange=timerange - ) - - ld_files = self.s() - if not data: - logger.critical("No data found. Terminating.") - return - # Use max_open_trades in backtesting, except --disable-max-market-positions is set - if self.config.get('use_max_market_positions', True): - max_open_trades = self.config['max_open_trades'] - else: - logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') - max_open_trades = 0 - - preprocessed = self.tickerdata_to_dataframe(data) - t_t = self.f(ld_files) - print("Load from json to file to df in mem took", t_t) - - # Print timeframe - min_date, max_date = self.get_timeframe(preprocessed) - logger.info( - 'Measuring data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days - ) - - # Execute backtest and print results - results = self.backtest( - { - 'stake_amount': self.config.get('stake_amount'), - 'processed': preprocessed, - 'max_open_trades': max_open_trades, - 'position_stacking': self.config.get('position_stacking', False), - } - ) - - if self.config.get('export', False): - self._store_backtest_result(self.config.get('exportfilename'), results) - - if self.use_backslap: - logger.info( - '\n====================================================== ' - 'BackSLAP REPORT' - ' =======================================================\n' - '%s', - self._generate_text_table( - data, - results - ) - ) - # optional print trades - if self.backslap_show_trades: - TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', - 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) - - def to_fwf(df, fname): - content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') - print(content) - - DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") - - # optional save trades - if self.backslap_save_trades: - TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', - 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) - - def to_fwf(df, fname): - content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') - open(fname, "w").write(content) - - DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") - - else: - logger.info( - '\n================================================= ' - 'BACKTEST REPORT' - ' ==================================================\n' - '%s', - self._generate_text_table( - data, - results - ) - ) - - if 'sell_reason' in results.columns: - logger.info( - '\n' + - ' SELL READON STATS '.center(119, '=') + - '\n%s \n', - self._generate_text_table_sell_reason(data, results) - - ) - else: - logger.info("no sell reasons available!") - - logger.info( - '\n' + - ' LEFT OPEN TRADES REPORT '.center(119, '=') + - '\n%s', - self._generate_text_table( - data, - results.loc[results.open_at_end] - ) - ) - - -def setup_configuration(args: Namespace) -> Dict[str, Any]: - """ - Prepare the configuration for the backtesting - :param args: Cli args from Arguments() - :return: Configuration - """ - configuration = Configuration(args) - config = configuration.get_config() - - # Ensure we do not use Exchange credentials - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - config['backslap'] = args.backslap - if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: - raise DependencyException('stake amount could not be "%s" for backtesting' % - constants.UNLIMITED_STAKE_AMOUNT) - - return config + def start(args: Namespace) -> None: diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 086cad5aa..508dc6bc8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -276,7 +276,7 @@ class Hyperopt(Backtesting): self.strategy.stoploss = params['stoploss'] processed = load(TICKERDATA_PICKLE) - results = self.backtest( + results = self.run( { 'stake_amount': self.config['stake_amount'], 'processed': processed, diff --git a/freqtrade/optimize/optimize.py b/freqtrade/optimize/optimize.py new file mode 100644 index 000000000..4e3d0ce6b --- /dev/null +++ b/freqtrade/optimize/optimize.py @@ -0,0 +1,322 @@ +# pragma pylint: disable=missing-docstring, W0212, too-many-arguments + +""" +This module contains the backtesting logic +""" +import logging +import operator +from abc import ABC, abstractmethod +from argparse import Namespace +from copy import deepcopy +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Dict, List, NamedTuple, Optional, Tuple + +import arrow +from pandas import DataFrame +from tabulate import tabulate + +from freqtrade import DependencyException, constants +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.exchange import Exchange +from freqtrade.misc import file_dump_json +import freqtrade.optimize as optimize +from freqtrade.strategy.interface import SellType +from freqtrade.strategy.resolver import IStrategy, StrategyResolver + +logger = logging.getLogger(__name__) + + +class BacktestResult(NamedTuple): + """ + NamedTuple Defining BacktestResults inputs. + """ + pair: str + profit_percent: float + profit_abs: float + open_time: datetime + close_time: datetime + open_index: int + close_index: int + trade_duration: float + open_at_end: bool + open_rate: float + close_rate: float + sell_reason: SellType + + +class IOptimize(ABC): + """ + Backtesting Abstract class, this class contains all the logic to run a backtest + + To run a backtest: + backtesting = Backtesting(config) + backtesting.start() + """ + + def __init__(self, config: Dict[str, Any]) -> None: + self.config = config + + # Reset keys for backtesting + self.config['exchange']['key'] = '' + self.config['exchange']['secret'] = '' + self.config['exchange']['password'] = '' + self.config['exchange']['uid'] = '' + self.config['dry_run'] = True + self.strategylist: List[IStrategy] = [] + if self.config.get('strategy_list', None): + # Force one interval + self.ticker_interval = str(self.config.get('ticker_interval')) + for strat in list(self.config['strategy_list']): + stratconf = deepcopy(self.config) + stratconf['strategy'] = strat + self.strategylist.append(StrategyResolver(stratconf).strategy) + + else: + # only one strategy + strat = StrategyResolver(self.config).strategy + + self.strategylist.append(StrategyResolver(self.config).strategy) + # Load one strategy + self._set_strategy(self.strategylist[0]) + + self.exchange = Exchange(self.config) + self.fee = self.exchange.get_fee() + + def _set_strategy(self, strategy): + """ + Load strategy into backtesting + """ + self.strategy = strategy + self.ticker_interval = self.config.get('ticker_interval') + self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe + self.advise_buy = strategy.advise_buy + self.advise_sell = strategy.advise_sell + + def _get_timeframe(self, data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: + """ + Get the maximum timeframe for the given backtest data + :param data: dictionary with preprocessed backtesting data + :return: tuple containing min_date, max_date + """ + timeframe = [ + (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) + for frame in data.values() + ] + return min(timeframe, key=operator.itemgetter(0))[0], \ + max(timeframe, key=operator.itemgetter(1))[1] + + def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: + """ + Generates and returns a text table for the given backtest data and the results dataframe + :return: pretty printed table with tabulate as str + """ + stake_currency = str(self.config.get('stake_currency')) + + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f') + tabular_data = [] + headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', + 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] + for pair in data: + result = results[results.pair == pair] + tabular_data.append([ + pair, + len(result.index), + result.profit_percent.mean() * 100.0, + result.profit_percent.sum() * 100.0, + result.profit_abs.sum(), + str(timedelta( + minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00', + len(result[result.profit_abs > 0]), + len(result[result.profit_abs < 0]) + ]) + + # Append Total + tabular_data.append([ + 'TOTAL', + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_percent.sum() * 100.0, + results.profit_abs.sum(), + str(timedelta( + minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', + len(results[results.profit_abs > 0]), + len(results[results.profit_abs < 0]) + ]) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") + + def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str: + """ + Generate small table outlining Backtest results + """ + tabular_data = [] + headers = ['Sell Reason', 'Count'] + for reason, count in results['sell_reason'].value_counts().iteritems(): + tabular_data.append([reason.value, count]) + return tabulate(tabular_data, headers=headers, tablefmt="pipe") + + def _generate_text_table_strategy(self, all_results: dict) -> str: + """ + Generate summary table per strategy + """ + stake_currency = str(self.config.get('stake_currency')) + + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f') + tabular_data = [] + headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %', + 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] + for strategy, results in all_results.items(): + tabular_data.append([ + strategy, + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_percent.sum() * 100.0, + results.profit_abs.sum(), + str(timedelta( + minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', + len(results[results.profit_abs > 0]), + len(results[results.profit_abs < 0]) + ]) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") + + def _store_backtest_result(self, recordfilename: str, results: DataFrame, + strategyname: Optional[str] = None) -> None: + + records = [(t.pair, t.profit_percent, t.open_time.timestamp(), + t.close_time.timestamp(), t.open_index - 1, t.trade_duration, + t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value) + for index, t in results.iterrows()] + + if records: + if strategyname: + # Inject strategyname to filename + recname = Path(recordfilename) + recordfilename = str(Path.joinpath( + recname.parent, f'{recname.stem}-{strategyname}').with_suffix(recname.suffix)) + logger.info('Dumping backtest results to %s', recordfilename) + file_dump_json(recordfilename, records) + + def start(self) -> None: + """ + Run a backtesting end-to-end + :return: None + """ + data = {} + pairs = self.config['exchange']['pair_whitelist'] + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + + if self.config.get('live'): + logger.info('Downloading data for all pairs in whitelist ...') + for pair in pairs: + data[pair] = self.exchange.get_candle_history(pair, self.ticker_interval) + else: + logger.info('Using local backtesting data (using whitelist in given config) ...') + + timerange = Arguments.parse_timerange(None if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) + data = optimize.load_data( + self.config['datadir'], + pairs=pairs, + ticker_interval=self.ticker_interval, + refresh_pairs=self.config.get('refresh_pairs', False), + exchange=self.exchange, + timerange=timerange + ) + + if not data: + logger.critical("No data found. Terminating.") + return + # Use max_open_trades in backtesting, except --disable-max-market-positions is set + if self.config.get('use_max_market_positions', True): + max_open_trades = self.config['max_open_trades'] + else: + logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') + max_open_trades = 0 + all_results = {} + + for strat in self.strategylist: + logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) + self._set_strategy(strat) + + # need to reprocess data every time to populate signals + preprocessed = self.tickerdata_to_dataframe(data) + + # Print timeframe + min_date, max_date = self._get_timeframe(preprocessed) + logger.info( + 'Measuring data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + + # Execute backtest and print results + all_results[self.strategy.get_strategy_name()] = self.run( + { + 'stake_amount': self.config.get('stake_amount'), + 'processed': preprocessed, + 'max_open_trades': max_open_trades, + 'position_stacking': self.config.get('position_stacking', False), + } + ) + + for strategy, results in all_results.items(): + + if self.config.get('export', False): + self._store_backtest_result(self.config['exportfilename'], results, + strategy if len(self.strategylist) > 1 else None) + + print(f"Result for strategy {strategy}") + print(' BACKTESTING REPORT '.center(119, '=')) + print(self._generate_text_table(data, results)) + + print(' SELL REASON STATS '.center(119, '=')) + print(self._generate_text_table_sell_reason(data, results)) + + print(' LEFT OPEN TRADES REPORT '.center(119, '=')) + print(self._generate_text_table(data, results.loc[results.open_at_end])) + print() + if len(all_results) > 1: + # Print Strategy summary table + print(' Strategy Summary '.center(119, '=')) + print(self._generate_text_table_strategy(all_results)) + print('\nFor more details, please look at the detail tables above') + + @abstractmethod + def run(self, args: Dict) -> DataFrame: + """ + Runs backtesting functionality. + + NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. + Of course try to not have ugly code. By some accessor are sometime slower than functions. + Avoid, logging on this method + + :param args: a dict containing: + stake_amount: btc amount to use for each trade + processed: a processed dictionary with format {pair, data} + max_open_trades: maximum number of concurrent trades (default: 0, disabled) + position_stacking: do we allow position stacking? (default: False) + :return: DataFrame + """ + + +def setup_configuration(args: Namespace) -> Dict[str, Any]: + """ + Prepare the configuration for the backtesting + :param args: Cli args from Arguments() + :return: Configuration + """ + configuration = Configuration(args) + config = configuration.get_config() + + # Ensure we do not use Exchange credentials + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + + if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: + raise DependencyException('stake amount could not be "%s" for backtesting' % + constants.UNLIMITED_STAKE_AMOUNT) + + return config From 6eb70cf75b285fc4917c89f7c96394e00f347048 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 14:30:14 +0200 Subject: [PATCH 50/51] set output depending on backtest/backslap type --- freqtrade/optimize/backslapping.py | 3 ++- freqtrade/optimize/backtesting.py | 3 ++- freqtrade/optimize/hyperopt.py | 2 ++ freqtrade/optimize/optimize.py | 9 ++++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backslapping.py b/freqtrade/optimize/backslapping.py index 6c8e1ae86..ef7596407 100644 --- a/freqtrade/optimize/backslapping.py +++ b/freqtrade/optimize/backslapping.py @@ -6,7 +6,7 @@ from typing import Dict, Any from pandas import DataFrame from freqtrade.exchange import Exchange -from freqtrade.optimize.optimize import IOptimize, BacktestResult, setup_configuration +from freqtrade.optimize.optimize import IOptimize, BacktestResult, OptimizeType, setup_configuration from freqtrade.strategy import IStrategy from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import StrategyResolver @@ -24,6 +24,7 @@ class Backslapping(IOptimize): constructor """ super().__init__(config) + self._optimizetype = OptimizeType.BACKTEST self.fee = self.exchange.get_fee() diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e4ed896be..12be5bad1 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List, Optional from pandas import DataFrame -from freqtrade.optimize.optimize import IOptimize, BacktestResult, setup_configuration +from freqtrade.optimize.optimize import IOptimize, BacktestResult, OptimizeType, setup_configuration from freqtrade.persistence import Trade from freqtrade.strategy.interface import SellType @@ -27,6 +27,7 @@ class Backtesting(IOptimize): def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) + self._optimizetype = OptimizeType.BACKTEST def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 508dc6bc8..11edf41fb 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -24,6 +24,7 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data +from freqtrade.optimize.optimize import OptimizeType from freqtrade.optimize.backtesting import Backtesting logger = logging.getLogger(__name__) @@ -42,6 +43,7 @@ class Hyperopt(Backtesting): """ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) + self._optimizetype = OptimizeType.HYPEROPT # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days self.target_trades = 600 diff --git a/freqtrade/optimize/optimize.py b/freqtrade/optimize/optimize.py index 4e3d0ce6b..18190478a 100644 --- a/freqtrade/optimize/optimize.py +++ b/freqtrade/optimize/optimize.py @@ -11,6 +11,7 @@ from copy import deepcopy from datetime import datetime, timedelta from pathlib import Path from typing import Any, Dict, List, NamedTuple, Optional, Tuple +from enum import Enum import arrow from pandas import DataFrame @@ -46,6 +47,12 @@ class BacktestResult(NamedTuple): sell_reason: SellType +class OptimizeType(Enum): + BACKTEST = "backtest" + BACKSLAP = "backslap" + HYPEROPT = "hyperopt" + + class IOptimize(ABC): """ Backtesting Abstract class, this class contains all the logic to run a backtest @@ -269,7 +276,7 @@ class IOptimize(ABC): strategy if len(self.strategylist) > 1 else None) print(f"Result for strategy {strategy}") - print(' BACKTESTING REPORT '.center(119, '=')) + print(f' {self._optimizetype.value.upper()} REPORT '.center(119, '=')) print(self._generate_text_table(data, results)) print(' SELL REASON STATS '.center(119, '=')) From 93091e9b22efd8ef0574605c4b9ed2b3d00198ff Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 14:30:33 +0200 Subject: [PATCH 51/51] fix tests for abstract class --- freqtrade/tests/optimize/test_backtesting.py | 30 ++++++++++---------- freqtrade/tests/optimize/test_hyperopt.py | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 32a5229c0..d0f13797d 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -91,7 +91,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None: data = load_data_test(contour) processed = backtesting.tickerdata_to_dataframe(data) assert isinstance(processed, dict) - results = backtesting.backtest( + results = backtesting.run( { 'stake_amount': config['stake_amount'], 'processed': processed, @@ -347,7 +347,7 @@ def test_get_timeframe(default_conf, mocker) -> None: pairs=['UNITTEST/BTC'] ) ) - min_date, max_date = backtesting.get_timeframe(data) + min_date, max_date = backtesting._get_timeframe(data) assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' assert max_date.isoformat() == '2017-11-14T22:58:00+00:00' @@ -459,9 +459,9 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', - backtest=MagicMock(), + run=MagicMock(), _generate_text_table=MagicMock(return_value='1'), - get_timeframe=get_timeframe, + _get_timeframe=get_timeframe, ) default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] @@ -494,9 +494,9 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', - backtest=MagicMock(), + run=MagicMock(), _generate_text_table=MagicMock(return_value='1'), - get_timeframe=get_timeframe, + _get_timeframe=get_timeframe, ) default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] @@ -521,7 +521,7 @@ def test_backtest(default_conf, fee, mocker) -> None: data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) data = trim_dictlist(data, -200) data_processed = backtesting.tickerdata_to_dataframe(data) - results = backtesting.backtest( + results = backtesting.run( { 'stake_amount': default_conf['stake_amount'], 'processed': data_processed, @@ -568,7 +568,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: # Run a backtesting for an exiting 5min ticker_interval data = optimize.load_data(None, ticker_interval='1m', pairs=['UNITTEST/BTC']) data = trim_dictlist(data, -200) - results = backtesting.backtest( + results = backtesting.run( { 'stake_amount': default_conf['stake_amount'], 'processed': backtesting.tickerdata_to_dataframe(data), @@ -612,7 +612,7 @@ def test_backtest_ticks(default_conf, fee, mocker): backtesting = Backtesting(default_conf) backtesting.advise_buy = fun # Override backtesting.advise_sell = fun # Override - results = backtesting.backtest(backtest_conf) + results = backtesting.run(backtest_conf) assert not results.empty @@ -627,7 +627,7 @@ def test_backtest_clash_buy_sell(mocker, default_conf): backtesting = Backtesting(default_conf) backtesting.advise_buy = fun # Override backtesting.advise_sell = fun # Override - results = backtesting.backtest(backtest_conf) + results = backtesting.run(backtest_conf) assert results.empty @@ -642,7 +642,7 @@ def test_backtest_only_sell(mocker, default_conf): backtesting = Backtesting(default_conf) backtesting.advise_buy = fun # Override backtesting.advise_sell = fun # Override - results = backtesting.backtest(backtest_conf) + results = backtesting.run(backtest_conf) assert results.empty @@ -652,7 +652,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker): backtesting = Backtesting(default_conf) backtesting.advise_buy = _trend_alternate # Override backtesting.advise_sell = _trend_alternate # Override - results = backtesting.backtest(backtest_conf) + results = backtesting.run(backtest_conf) backtesting._store_backtest_result("test_.json", results) assert len(results) == 4 # One trade was force-closed at the end @@ -665,7 +665,7 @@ def test_backtest_record(default_conf, fee, mocker): patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch( - 'freqtrade.optimize.backtesting.file_dump_json', + 'freqtrade.optimize.optimize.file_dump_json', new=lambda n, r: (names.append(n), records.append(r)) ) @@ -736,7 +736,7 @@ def test_backtest_start_live(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange.get_candle_history', new=lambda s, n, i: _load_pair_as_ticks(n, i)) patch_exchange(mocker) - mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.run', MagicMock()) mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock()) mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) @@ -780,7 +780,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog): new=lambda s, n, i: _load_pair_as_ticks(n, i)) patch_exchange(mocker) backtestmock = MagicMock() - mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.run', backtestmock) gen_table_mock = MagicMock() mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', gen_table_mock) gen_strattable_mock = MagicMock() diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 65a3c2fdb..12282c1a9 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -263,7 +263,7 @@ def test_generate_optimizer(mocker, default_conf) -> None: backtest_result = pd.DataFrame.from_records(trades, columns=labels) mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.backtest', + 'freqtrade.optimize.hyperopt.Hyperopt.run', MagicMock(return_value=backtest_result) ) patch_exchange(mocker)