From a6d74a146363e60d918b291f819d5de65ea57ccd Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 30 Nov 2018 20:42:16 +0100 Subject: [PATCH 01/49] Draft of dataprovider --- freqtrade/dataprovider.py | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 freqtrade/dataprovider.py diff --git a/freqtrade/dataprovider.py b/freqtrade/dataprovider.py new file mode 100644 index 000000000..0220048c6 --- /dev/null +++ b/freqtrade/dataprovider.py @@ -0,0 +1,45 @@ +""" +Dataprovider +Responsible to provide data to the bot +including Klines, tickers, historic data +Common Interface for bot and strategy to access data. +""" +import logging + +from freqtrade.exchange import Exchange + +logger = logging.getLogger(__name__) + + +class DataProvider(object): + + def __init__(self, exchange: Exchange) -> None: + pass + + def refresh() -> None: + """ + Refresh data, called with each cycle + """ + pass + + def kline(pair: str): + """ + get ohlcv data for the given pair + """ + pass + + def historic_kline(pair: str): + """ + get historic ohlcv data stored for backtesting + """ + pass + + def ticker(pair: str): + pass + + def orderbook(pair: str, max: int): + pass + + def balance(pair): + # TODO: maybe use wallet directly?? + pass From b119a767de5de4d66a2912c3b68a88d6cd19325d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Dec 2018 09:16:35 +0100 Subject: [PATCH 02/49] Some more restructuring --- freqtrade/dataprovider.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/freqtrade/dataprovider.py b/freqtrade/dataprovider.py index 0220048c6..4b924bd9a 100644 --- a/freqtrade/dataprovider.py +++ b/freqtrade/dataprovider.py @@ -6,6 +6,8 @@ Common Interface for bot and strategy to access data. """ import logging +from pandas import DataFrame + from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) @@ -13,8 +15,9 @@ logger = logging.getLogger(__name__) class DataProvider(object): - def __init__(self, exchange: Exchange) -> None: - pass + def __init__(self, config: dict, exchange: Exchange) -> None: + self._config = config + self._exchange = exchange def refresh() -> None: """ @@ -22,24 +25,31 @@ class DataProvider(object): """ pass - def kline(pair: str): + def ohlcv(self, pair: str) -> DataFrame: """ - get ohlcv data for the given pair + get ohlcv data for the given pair as DataFrame """ - pass + # TODO: Should not be stored in exchange but in this class + return self._exchange.klines.get(pair) - def historic_kline(pair: str): + def historic_ohlcv(self, pair: str) -> DataFrame: """ get historic ohlcv data stored for backtesting """ pass - def ticker(pair: str): + def ticker(self, pair: str): + """ + Return last ticker data + """ pass - def orderbook(pair: str, max: int): + def orderbook(self, pair: str, max: int): + """ + return latest orderbook data + """ pass - def balance(pair): + def balance(self, pair): # TODO: maybe use wallet directly?? pass From 7206287b001d561a2e32ee412bdd294501d9dd58 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Dec 2018 15:57:49 +0100 Subject: [PATCH 03/49] Use Dataprovider --- freqtrade/dataprovider.py | 7 ++++--- freqtrade/freqtradebot.py | 9 ++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/freqtrade/dataprovider.py b/freqtrade/dataprovider.py index 4b924bd9a..035e9852a 100644 --- a/freqtrade/dataprovider.py +++ b/freqtrade/dataprovider.py @@ -5,6 +5,7 @@ including Klines, tickers, historic data Common Interface for bot and strategy to access data. """ import logging +from typing import List, Dict from pandas import DataFrame @@ -19,13 +20,13 @@ class DataProvider(object): self._config = config self._exchange = exchange - def refresh() -> None: + def refresh(self, pairlist: List[str]) -> None: """ Refresh data, called with each cycle """ - pass + self._exchange.refresh_tickers(pairlist, self._config['ticker_interval']) - def ohlcv(self, pair: str) -> DataFrame: + def ohlcv(self, pair: str) -> List[str]: """ get ohlcv data for the given pair as DataFrame """ diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2e09cf116..9bcc3f86b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -16,6 +16,7 @@ from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.data.converter import order_book_to_dataframe from freqtrade.edge import Edge +from freqtrade.dataprovider import DataProvider from freqtrade.exchange import Exchange from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType @@ -57,6 +58,8 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) + + self.dataprovider = DataProvider(self.config, self.exchange) pairlistname = self.config.get('pairlist', {}).get('method', 'StaticPairList') self.pairlists = PairListResolver(pairlistname, self, self.config).pairlist @@ -167,7 +170,7 @@ class FreqtradeBot(object): if trade.pair not in self.active_pair_whitelist]) # Refreshing candles - self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) + self.dataprovider.refresh(self.active_pair_whitelist) # First process current opened trades for trade in trades: @@ -317,7 +320,7 @@ class FreqtradeBot(object): # running get_signal on historical data fetched for _pair in whitelist: - (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines(_pair)) + (buy, sell) = self.strategy.get_signal(_pair, interval, self.dataprovider.ohlcv(_pair)) if buy and not sell: stake_amount = self._get_trade_stake_amount(_pair) if not stake_amount: @@ -579,7 +582,7 @@ class FreqtradeBot(object): experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, - self.exchange.klines(trade.pair)) + self.dataprovider.ohlcv(trade.pair)) config_ask_strategy = self.config.get('ask_strategy', {}) if config_ask_strategy.get('use_order_book', False): From 05570732c6d3561d2aaac54b2e8967a17cd2eda2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 2 Dec 2018 21:57:30 +0100 Subject: [PATCH 04/49] add get_runmode --- freqtrade/dataprovider.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/freqtrade/dataprovider.py b/freqtrade/dataprovider.py index 035e9852a..ddbd35fb7 100644 --- a/freqtrade/dataprovider.py +++ b/freqtrade/dataprovider.py @@ -31,6 +31,7 @@ class DataProvider(object): get ohlcv data for the given pair as DataFrame """ # TODO: Should not be stored in exchange but in this class + # TODO: should return dataframe, not list return self._exchange.klines.get(pair) def historic_ohlcv(self, pair: str) -> DataFrame: @@ -54,3 +55,11 @@ class DataProvider(object): def balance(self, pair): # TODO: maybe use wallet directly?? pass + + @property + def runmode(self) -> str: + """ + Get runmode of the bot + can be "live", "dry-run", "backtest", "edgecli", "hyperopt". + """ + return self._config.get['runmode'] From 4ab7edd3d6f412abbc0f46dd3581cfa3ecdec4ba Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 17 Dec 2018 06:43:01 +0100 Subject: [PATCH 05/49] small adaptations --- freqtrade/{ => data}/dataprovider.py | 2 +- freqtrade/freqtradebot.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename freqtrade/{ => data}/dataprovider.py (97%) diff --git a/freqtrade/dataprovider.py b/freqtrade/data/dataprovider.py similarity index 97% rename from freqtrade/dataprovider.py rename to freqtrade/data/dataprovider.py index ddbd35fb7..32969fe3b 100644 --- a/freqtrade/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -32,7 +32,7 @@ class DataProvider(object): """ # TODO: Should not be stored in exchange but in this class # TODO: should return dataframe, not list - return self._exchange.klines.get(pair) + return self._exchange.klines(pair) def historic_ohlcv(self, pair: str) -> DataFrame: """ diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9bcc3f86b..4d0369849 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -15,8 +15,8 @@ from requests.exceptions import RequestException from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.data.converter import order_book_to_dataframe +from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.dataprovider import DataProvider from freqtrade.exchange import Exchange from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType From f1a5a8e20ee021650e365d144de055d4ec4bde01 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 17 Dec 2018 06:52:13 +0100 Subject: [PATCH 06/49] provide history --- freqtrade/data/dataprovider.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 32969fe3b..5970e826f 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -5,11 +5,13 @@ including Klines, tickers, historic data Common Interface for bot and strategy to access data. """ import logging +from pathlib import Path from typing import List, Dict from pandas import DataFrame from freqtrade.exchange import Exchange +from freqtrade.data.history import load_pair_history logger = logging.getLogger(__name__) @@ -31,14 +33,18 @@ class DataProvider(object): get ohlcv data for the given pair as DataFrame """ # TODO: Should not be stored in exchange but in this class - # TODO: should return dataframe, not list return self._exchange.klines(pair) - def historic_ohlcv(self, pair: str) -> DataFrame: + def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame: """ get historic ohlcv data stored for backtesting """ - pass + return load_pair_history(pair=pair, + ticker_interval=ticker_interval, + refresh_pairs=False, + datadir=Path(self.config['datadir']) if self.config.get( + 'datadir') else None + ) def ticker(self, pair: str): """ @@ -62,4 +68,5 @@ class DataProvider(object): Get runmode of the bot can be "live", "dry-run", "backtest", "edgecli", "hyperopt". """ + # TODO: this needs to be set somewhere ... return self._config.get['runmode'] From e38c06afe98c8b6405d6eab4802e46d5dd56e4b6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Dec 2018 13:20:25 +0100 Subject: [PATCH 07/49] Small fixes --- freqtrade/data/dataprovider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 5970e826f..142c12f2a 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -42,7 +42,7 @@ class DataProvider(object): return load_pair_history(pair=pair, ticker_interval=ticker_interval, refresh_pairs=False, - datadir=Path(self.config['datadir']) if self.config.get( + datadir=Path(self._config['datadir']) if self._config.get( 'datadir') else None ) @@ -69,4 +69,4 @@ class DataProvider(object): can be "live", "dry-run", "backtest", "edgecli", "hyperopt". """ # TODO: this needs to be set somewhere ... - return self._config.get['runmode'] + return str(self._config.get('runmode')) From 84cc4887cedd45b182c622c3e0a7bf3c39666381 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Dec 2018 13:37:15 +0100 Subject: [PATCH 08/49] Add copy parameter --- freqtrade/data/dataprovider.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 142c12f2a..4d819a7d8 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -28,12 +28,15 @@ class DataProvider(object): """ self._exchange.refresh_tickers(pairlist, self._config['ticker_interval']) - def ohlcv(self, pair: str) -> List[str]: + def ohlcv(self, pair: str, copy: bool = True) -> List[str]: """ get ohlcv data for the given pair as DataFrame + :param pair: pair to get the data for + :param copy: copy dataframe before returning. + Use false only for RO operations (where the dataframe is not modified) """ # TODO: Should not be stored in exchange but in this class - return self._exchange.klines(pair) + return self._exchange.klines(pair, copy) def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame: """ From a7db4d74cb0a7b38159cab1d6586506e29cbd510 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Dec 2018 13:56:30 +0100 Subject: [PATCH 09/49] Add some simple dataprovider tests --- freqtrade/tests/data/test_dataprovider.py | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 freqtrade/tests/data/test_dataprovider.py diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py new file mode 100644 index 000000000..3eaa45abb --- /dev/null +++ b/freqtrade/tests/data/test_dataprovider.py @@ -0,0 +1,36 @@ +from unittest.mock import Mock, MagicMock, PropertyMock + +from pandas import DataFrame + +from freqtrade.data.dataprovider import DataProvider +from freqtrade.exchange import Exchange +from freqtrade.tests.conftest import get_patched_exchange, log_has + + +def test_ohlcv(mocker, default_conf, ticker_history): + + exchange = get_patched_exchange(mocker, default_conf) + exchange._klines['XRP/BTC'] = ticker_history + exchange._klines['UNITEST/BTC'] = ticker_history + dp = DataProvider(default_conf, exchange) + assert ticker_history.equals(dp.ohlcv('UNITEST/BTC')) + assert isinstance(dp.ohlcv('UNITEST/BTC'), DataFrame) + assert dp.ohlcv('UNITEST/BTC') is not ticker_history + assert dp.ohlcv('UNITEST/BTC', copy=False) is ticker_history + assert dp.ohlcv('NONESENSE/AAA') is None + + +def test_historic_ohlcv(mocker, default_conf, ticker_history): + + historymock = MagicMock(return_value=ticker_history) + mocker.patch('freqtrade.data.dataprovider.load_pair_history', historymock) + + # exchange = get_patched_exchange(mocker, default_conf) + dp = DataProvider(default_conf, None) + data = dp.historic_ohlcv('UNITTEST/BTC', "5m") + assert isinstance(data, DataFrame) + assert historymock.call_count == 1 + assert historymock.call_args_list[0][1]['datadir'] is None + assert historymock.call_args_list[0][1]['refresh_pairs'] == False + assert historymock.call_args_list[0][1]['ticker_interval'] == '5m' + From fed3ebfb468d4ce28346eb9860ba8449e202b122 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Dec 2018 14:05:40 +0100 Subject: [PATCH 10/49] Change enum from 0 to 1 according to the documentation see [here](https://docs.python.org/3/library/enum.html#functional-api) --- freqtrade/state.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/state.py b/freqtrade/state.py index 42bfb6e41..4845b72f0 100644 --- a/freqtrade/state.py +++ b/freqtrade/state.py @@ -3,13 +3,13 @@ """ Bot state constant """ -import enum +from enum import Enum -class State(enum.Enum): +class State(Enum): """ Bot application states """ - RUNNING = 0 - STOPPED = 1 - RELOAD_CONF = 2 + RUNNING = 1 + STOPPED = 2 + RELOAD_CONF = 3 From 1340b71633b455fee16b88892b179cc54abdbe92 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Dec 2018 14:23:59 +0100 Subject: [PATCH 11/49] Add RunMode setting to determine bot state --- freqtrade/configuration.py | 11 ++++++++++- freqtrade/data/dataprovider.py | 10 +++++----- freqtrade/main.py | 4 ++-- freqtrade/optimize/backtesting.py | 3 ++- freqtrade/optimize/edge_cli.py | 5 +++-- freqtrade/optimize/hyperopt.py | 3 ++- freqtrade/state.py | 13 +++++++++++++ freqtrade/tests/data/test_dataprovider.py | 8 +++----- scripts/plot_profit.py | 3 ++- 9 files changed, 42 insertions(+), 18 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 9fd93629f..d972f50b8 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -12,6 +12,7 @@ from jsonschema import Draft4Validator, validate from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants +from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -34,9 +35,10 @@ class Configuration(object): Reuse this class for the bot, backtesting, hyperopt and every script that required configuration """ - def __init__(self, args: Namespace) -> None: + def __init__(self, args: Namespace, runmode: RunMode = None) -> None: self.args = args self.config: Optional[Dict[str, Any]] = None + self.runmode = runmode def load_config(self) -> Dict[str, Any]: """ @@ -68,6 +70,13 @@ class Configuration(object): # Load Hyperopt config = self._load_hyperopt_config(config) + # Set runmode + if not self.runmode: + # Handle real mode, infer dry/live from config + self.runmode = RunMode.DRY_RUN if config.get('dry_run', True) else RunMode.LIVE + + config.update({'runmode': self.runmode}) + return config def _load_config_file(self, path: str) -> Dict[str, Any]: diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 4d819a7d8..5df2f2fd9 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -6,12 +6,13 @@ Common Interface for bot and strategy to access data. """ import logging from pathlib import Path -from typing import List, Dict +from typing import List from pandas import DataFrame -from freqtrade.exchange import Exchange from freqtrade.data.history import load_pair_history +from freqtrade.exchange import Exchange +from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -66,10 +67,9 @@ class DataProvider(object): pass @property - def runmode(self) -> str: + def runmode(self) -> RunMode: """ Get runmode of the bot can be "live", "dry-run", "backtest", "edgecli", "hyperopt". """ - # TODO: this needs to be set somewhere ... - return str(self._config.get('runmode')) + return self._config.get('runmode') diff --git a/freqtrade/main.py b/freqtrade/main.py index f27145b45..75b15915b 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -39,7 +39,7 @@ def main(sysargv: List[str]) -> None: return_code = 1 try: # Load and validate configuration - config = Configuration(args).get_config() + config = Configuration(args, None).get_config() # Init the bot freqtrade = FreqtradeBot(config) @@ -76,7 +76,7 @@ def reconfigure(freqtrade: FreqtradeBot, args: Namespace) -> FreqtradeBot: freqtrade.cleanup() # Create new instance - freqtrade = FreqtradeBot(Configuration(args).get_config()) + freqtrade = FreqtradeBot(Configuration(args, None).get_config()) freqtrade.rpc.send_msg({ 'type': RPCMessageType.STATUS_NOTIFICATION, 'status': 'config reloaded' diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 38bbe13d4..88ad4cc60 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -22,6 +22,7 @@ from freqtrade.data import history from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from freqtrade.resolvers import StrategyResolver +from freqtrade.state import RunMode from freqtrade.strategy.interface import SellType, IStrategy logger = logging.getLogger(__name__) @@ -452,7 +453,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]: :param args: Cli args from Arguments() :return: Configuration """ - configuration = Configuration(args) + configuration = Configuration(args, RunMode.BACKTEST) config = configuration.get_config() # Ensure we do not use Exchange credentials diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index fdae47b99..9b628cf2e 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -9,10 +9,11 @@ from typing import Dict, Any from tabulate import tabulate from freqtrade.edge import Edge -from freqtrade.configuration import Configuration from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.resolvers import StrategyResolver +from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -83,7 +84,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]: :param args: Cli args from Arguments() :return: Configuration """ - configuration = Configuration(args) + configuration = Configuration(args, RunMode.EDGECLI) config = configuration.get_config() # Ensure we do not use Exchange credentials diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 6930bed04..f6d39f11c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -25,6 +25,7 @@ from freqtrade.configuration import Configuration from freqtrade.data.history import load_data from freqtrade.optimize import get_timeframe from freqtrade.optimize.backtesting import Backtesting +from freqtrade.state import RunMode from freqtrade.resolvers import HyperOptResolver logger = logging.getLogger(__name__) @@ -306,7 +307,7 @@ def start(args: Namespace) -> None: # Initialize configuration # Monkey patch the configuration with hyperopt_conf.py - configuration = Configuration(args) + configuration = Configuration(args, RunMode.HYPEROPT) logger.info('Starting freqtrade in Hyperopt mode') config = configuration.load_config() diff --git a/freqtrade/state.py b/freqtrade/state.py index 4845b72f0..b69c26cb5 100644 --- a/freqtrade/state.py +++ b/freqtrade/state.py @@ -13,3 +13,16 @@ class State(Enum): RUNNING = 1 STOPPED = 2 RELOAD_CONF = 3 + + +class RunMode(Enum): + """ + Bot running mode (backtest, hyperopt, ...) + can be "live", "dry-run", "backtest", "edgecli", "hyperopt". + """ + LIVE = "live" + DRY_RUN = "dry_run" + BACKTEST = "backtest" + EDGECLI = "edgecli" + HYPEROPT = "hyperopt" + OTHER = "other" # Used for plotting scripts and test diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py index 3eaa45abb..154ef07a5 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/freqtrade/tests/data/test_dataprovider.py @@ -1,10 +1,9 @@ -from unittest.mock import Mock, MagicMock, PropertyMock +from unittest.mock import MagicMock from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider -from freqtrade.exchange import Exchange -from freqtrade.tests.conftest import get_patched_exchange, log_has +from freqtrade.tests.conftest import get_patched_exchange def test_ohlcv(mocker, default_conf, ticker_history): @@ -31,6 +30,5 @@ def test_historic_ohlcv(mocker, default_conf, ticker_history): assert isinstance(data, DataFrame) assert historymock.call_count == 1 assert historymock.call_args_list[0][1]['datadir'] is None - assert historymock.call_args_list[0][1]['refresh_pairs'] == False + assert historymock.call_args_list[0][1]['refresh_pairs'] is False assert historymock.call_args_list[0][1]['ticker_interval'] == '5m' - diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index a1561bc89..72ac4031a 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -29,6 +29,7 @@ from freqtrade.configuration import Configuration from freqtrade import constants from freqtrade.data import history from freqtrade.resolvers import StrategyResolver +from freqtrade.state import RunMode import freqtrade.misc as misc @@ -82,7 +83,7 @@ def plot_profit(args: Namespace) -> None: # to match the tickerdata against the profits-results timerange = Arguments.parse_timerange(args.timerange) - config = Configuration(args).get_config() + config = Configuration(args, RunMode.OTHER).get_config() # Init strategy try: From f034235af4dac69e48429cb31328e89197815f97 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Dec 2018 14:35:48 +0100 Subject: [PATCH 12/49] Tests for RunMode --- freqtrade/data/dataprovider.py | 2 +- freqtrade/tests/optimize/test_backtesting.py | 5 +++++ freqtrade/tests/optimize/test_edge_cli.py | 4 ++++ freqtrade/tests/test_configuration.py | 12 ++++++++++-- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 5df2f2fd9..dbca1e035 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -72,4 +72,4 @@ class DataProvider(object): Get runmode of the bot can be "live", "dry-run", "backtest", "edgecli", "hyperopt". """ - return self._config.get('runmode') + return RunMode(self._config.get('runmode', RunMode.OTHER)) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 5ab44baad..beef1b16e 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -18,6 +18,7 @@ from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.optimize import get_timeframe from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, start) +from freqtrade.state import RunMode from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import SellType from freqtrade.tests.conftest import log_has, patch_exchange @@ -200,6 +201,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'timerange' not in config assert 'export' not in config + assert 'runmode' in config + assert config['runmode'] == RunMode.BACKTEST def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> None: @@ -230,6 +233,8 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert 'exchange' in config assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config + assert config['runmode'] == RunMode.BACKTEST + assert log_has( 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples diff --git a/freqtrade/tests/optimize/test_edge_cli.py b/freqtrade/tests/optimize/test_edge_cli.py index 8ffab7f11..a58620139 100644 --- a/freqtrade/tests/optimize/test_edge_cli.py +++ b/freqtrade/tests/optimize/test_edge_cli.py @@ -7,6 +7,7 @@ from typing import List from freqtrade.edge import PairInfo from freqtrade.arguments import Arguments from freqtrade.optimize.edge_cli import (EdgeCli, setup_configuration, start) +from freqtrade.state import RunMode from freqtrade.tests.conftest import log_has, patch_exchange @@ -26,6 +27,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> ] config = setup_configuration(get_args(args)) + assert config['runmode'] == RunMode.EDGECLI + assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -70,6 +73,7 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N assert 'exchange' in config assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config + assert config['runmode'] == RunMode.EDGECLI assert log_has( 'Using data folder: {} ...'.format(config['datadir']), caplog.record_tuples diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index f5c887089..67445238b 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -13,6 +13,7 @@ from freqtrade import OperationalException from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration, set_loggers from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL +from freqtrade.state import RunMode from freqtrade.tests.conftest import log_has @@ -77,6 +78,8 @@ def test_load_config_max_open_trades_minus_one(default_conf, mocker, caplog) -> assert validated_conf['max_open_trades'] > 999999999 assert validated_conf['max_open_trades'] == float('inf') assert log_has('Validating configuration ...', caplog.record_tuples) + assert "runmode" in validated_conf + assert validated_conf['runmode'] == RunMode.DRY_RUN def test_load_config_file_exception(mocker) -> None: @@ -177,6 +180,8 @@ def test_load_config_with_params(default_conf, mocker) -> None: configuration = Configuration(args) validated_conf = configuration.load_config() assert validated_conf.get('db_url') == DEFAULT_DB_PROD_URL + assert "runmode" in validated_conf + assert validated_conf['runmode'] == RunMode.LIVE # Test args provided db_url dry_run conf = default_conf.copy() @@ -365,8 +370,9 @@ def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> Non args = Arguments(arglist, '').get_parsed_arg() - configuration = Configuration(args) + configuration = Configuration(args, RunMode.BACKTEST) config = configuration.get_config() + assert config['runmode'] == RunMode.BACKTEST assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -411,7 +417,7 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: ] args = Arguments(arglist, '').get_parsed_arg() - configuration = Configuration(args) + configuration = Configuration(args, RunMode.HYPEROPT) config = configuration.get_config() assert 'epochs' in config @@ -422,6 +428,8 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: assert 'spaces' in config assert config['spaces'] == ['all'] assert log_has('Parameter -s/--spaces detected: [\'all\']', caplog.record_tuples) + assert "runmode" in config + assert config['runmode'] == RunMode.HYPEROPT def test_check_exchange(default_conf, caplog) -> None: From d3a37db79ada8a9802488df419883995b34d7d60 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Dec 2018 14:23:21 +0100 Subject: [PATCH 13/49] Provide available pairs --- freqtrade/data/dataprovider.py | 8 ++++++++ freqtrade/tests/data/test_dataprovider.py | 20 +++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index dbca1e035..31489d43c 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -29,6 +29,14 @@ class DataProvider(object): """ self._exchange.refresh_tickers(pairlist, self._config['ticker_interval']) + @property + def available_pairs(self) -> List[str]: + """ + Return a list of pairs for which data is currently cached. + Should be whitelist + open trades. + """ + return list(self._exchange._klines.keys()) + def ohlcv(self, pair: str, copy: bool = True) -> List[str]: """ get ohlcv data for the given pair as DataFrame diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py index 154ef07a5..86fcf3699 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/freqtrade/tests/data/test_dataprovider.py @@ -10,12 +10,12 @@ def test_ohlcv(mocker, default_conf, ticker_history): exchange = get_patched_exchange(mocker, default_conf) exchange._klines['XRP/BTC'] = ticker_history - exchange._klines['UNITEST/BTC'] = ticker_history + exchange._klines['UNITTEST/BTC'] = ticker_history dp = DataProvider(default_conf, exchange) - assert ticker_history.equals(dp.ohlcv('UNITEST/BTC')) - assert isinstance(dp.ohlcv('UNITEST/BTC'), DataFrame) - assert dp.ohlcv('UNITEST/BTC') is not ticker_history - assert dp.ohlcv('UNITEST/BTC', copy=False) is ticker_history + assert ticker_history.equals(dp.ohlcv('UNITTEST/BTC')) + assert isinstance(dp.ohlcv('UNITTEST/BTC'), DataFrame) + assert dp.ohlcv('UNITTEST/BTC') is not ticker_history + assert dp.ohlcv('UNITTEST/BTC', copy=False) is ticker_history assert dp.ohlcv('NONESENSE/AAA') is None @@ -32,3 +32,13 @@ def test_historic_ohlcv(mocker, default_conf, ticker_history): assert historymock.call_args_list[0][1]['datadir'] is None assert historymock.call_args_list[0][1]['refresh_pairs'] is False assert historymock.call_args_list[0][1]['ticker_interval'] == '5m' + + +def test_available_pairs(mocker, default_conf, ticker_history): + exchange = get_patched_exchange(mocker, default_conf) + exchange._klines['XRP/BTC'] = ticker_history + exchange._klines['UNITTEST/BTC'] = ticker_history + dp = DataProvider(default_conf, exchange) + + assert len(dp.available_pairs) == 2 + assert dp.available_pairs == ['XRP/BTC', 'UNITTEST/BTC'] From 58f1abf28712cb3f3c8aa3d4ff05b26f0c5dc5dc Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Dec 2018 14:32:17 +0100 Subject: [PATCH 14/49] Add dp / wallets to strategy interface --- freqtrade/freqtradebot.py | 7 ++++++- freqtrade/strategy/interface.py | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 4d0369849..a096a37da 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -58,8 +58,13 @@ class FreqtradeBot(object): self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) - self.dataprovider = DataProvider(self.config, self.exchange) + + # Attach Dataprovider to Strategy baseclass + IStrategy.dp = self.dataprovider + # Attach Wallets to Strategy baseclass + IStrategy.wallets = self.wallets + pairlistname = self.config.get('pairlist', {}).get('method', 'StaticPairList') self.pairlists = PairListResolver(pairlistname, self, self.config).pairlist diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 08a5cf1cd..7210f5c78 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -13,7 +13,9 @@ import arrow from pandas import DataFrame from freqtrade import constants +from freqtrade.data.dataprovider import DataProvider from freqtrade.persistence import Trade +from freqtrade.wallets import Wallets logger = logging.getLogger(__name__) @@ -96,6 +98,12 @@ class IStrategy(ABC): # Dict to determine if analysis is necessary _last_candle_seen_per_pair: Dict[str, datetime] = {} + # Class level variables (intentional) containing + # the dataprovider (dp) (access to other candles, historic data, ...) + # and wallets - access to the current balance. + dp: DataProvider + wallets: Wallets + def __init__(self, config: dict) -> None: self.config = config self._last_candle_seen_per_pair = {} From 5ecdecd1ebd852e1a6fb99b34973650878724f02 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Dec 2018 14:37:25 +0100 Subject: [PATCH 15/49] remove unused local variable persistance --- freqtrade/freqtradebot.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a096a37da..071cbaf2e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -37,9 +37,9 @@ class FreqtradeBot(object): def __init__(self, config: Dict[str, Any])-> None: """ - Init all variables and object the bot need to work - :param config: configuration dict, you can use the Configuration.get_config() - method to get the config dict. + Init all variables and objects the bot needs to work + :param config: configuration dict, you can use Configuration.get_config() + to get the config dict. """ logger.info( @@ -55,7 +55,6 @@ class FreqtradeBot(object): self.strategy: IStrategy = StrategyResolver(self.config).strategy self.rpc: RPCManager = RPCManager(self) - self.persistence = None self.exchange = Exchange(self.config) self.wallets = Wallets(self.exchange) self.dataprovider = DataProvider(self.config, self.exchange) From 8f3ea3608adbcc45616e625d399bdcb600e06e7c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 Dec 2018 14:58:16 +0100 Subject: [PATCH 16/49] some cleanup --- freqtrade/data/dataprovider.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 31489d43c..758e401e8 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -62,16 +62,14 @@ class DataProvider(object): """ Return last ticker data """ + # TODO: Implement me pass def orderbook(self, pair: str, max: int): """ return latest orderbook data """ - pass - - def balance(self, pair): - # TODO: maybe use wallet directly?? + # TODO: Implement me pass @property From 35c8d1dcbe85441cb89f4be2fafcbefa5b144824 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Dec 2018 08:47:14 +0100 Subject: [PATCH 17/49] Update comment --- freqtrade/data/dataprovider.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 758e401e8..c9c3e452a 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -40,6 +40,7 @@ class DataProvider(object): def ohlcv(self, pair: str, copy: bool = True) -> List[str]: """ get ohlcv data for the given pair as DataFrame + Please check `available_pairs` to verify which pairs are currently cached. :param pair: pair to get the data for :param copy: copy dataframe before returning. Use false only for RO operations (where the dataframe is not modified) From 9edb88051d8291ab33eb199a6c449a9095e3b62d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Dec 2018 08:47:25 +0100 Subject: [PATCH 18/49] Add dataprovider documentation --- docs/bot-optimization.md | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 1cfae1bc4..6d03f55ac 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -222,6 +222,57 @@ Please note that the same buy/sell signals may work with one interval, but not t The metadata-dict (available for `populate_buy_trend`, `populate_sell_trend`, `populate_indicators`) contains additional information. Currently this is `pair`, which can be accessed using `metadata['pair']` - and will return a pair in the format `XRP/BTC`. +### Additional data (DataProvider) + +The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. + +**NOTE**: The DataProvier is currently not available during backtesting / hyperopt. + +Please always check if the `DataProvider` is available to avoid failures during backtesting. + +``` python +if self.dp: + if dp.runmode == 'live': + if 'ETH/BTC' in self.dp.available_pairs: + data_eth = self.dp.ohlcv(pair='ETH/BTC', + ticker_interval=ticker_interval) + else: + # Get historic ohlcv data (cached on disk). + history_eth = self.dp.historic_ohlcv(pair='ETH/BTC', + ticker_interval='1h') +``` + +All methods return `None` in case of failure (do not raise an exception). + +#### Possible options for DataProvider + +- `available_pairs` - Property containing cached pairs +- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for all pairs in the whitelist +- `historic_ohlcv(pair, ticker_interval)` - Data stored on disk +- `runmode` - Property containing the current runmode. + +### Additional data - Wallets + +The strategy provides access to the `Wallets` object. This contains the current balances on the exchange. + +**NOTE**: Wallets is not available during backtesting / hyperopt. + +Please always check if `Wallets` is available to avoid failures during backtesting. + +``` python +if self.wallets: + free_eth = self.wallets.get_free('ETH') + used_eth = self.wallets.get_used('ETH') + total_eth = self.wallets.get_total('ETH') +``` + +#### Possible options for Wallets + +- `get_free(asset)` - currently available balance to trade +- `get_used(asset)` - currently tied up balance (open orders) +- `get_total(asset)` - total available balance - sum of the 2 above + + ### Where is the default strategy? The default buy strategy is located in the file From 2b029b2a8627be34d02ec6025fd43fed473d8b4c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Dec 2018 09:13:20 +0100 Subject: [PATCH 19/49] Only return ohlcv if available (Live and dry modes) --- freqtrade/data/dataprovider.py | 7 +++++-- freqtrade/tests/data/test_dataprovider.py | 14 +++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index c9c3e452a..e75f6ce33 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -46,7 +46,10 @@ class DataProvider(object): Use false only for RO operations (where the dataframe is not modified) """ # TODO: Should not be stored in exchange but in this class - return self._exchange.klines(pair, copy) + if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): + return self._exchange.klines(pair, copy) + else: + return None def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame: """ @@ -77,6 +80,6 @@ class DataProvider(object): def runmode(self) -> RunMode: """ Get runmode of the bot - can be "live", "dry-run", "backtest", "edgecli", "hyperopt". + can be "live", "dry-run", "backtest", "edgecli", "hyperopt" or "other". """ return RunMode(self._config.get('runmode', RunMode.OTHER)) diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py index 86fcf3699..9988d0fef 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/freqtrade/tests/data/test_dataprovider.py @@ -3,21 +3,33 @@ from unittest.mock import MagicMock from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider +from freqtrade.state import RunMode from freqtrade.tests.conftest import get_patched_exchange def test_ohlcv(mocker, default_conf, ticker_history): - + default_conf['runmode'] = RunMode.DRY_RUN exchange = get_patched_exchange(mocker, default_conf) exchange._klines['XRP/BTC'] = ticker_history exchange._klines['UNITTEST/BTC'] = ticker_history dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.DRY_RUN assert ticker_history.equals(dp.ohlcv('UNITTEST/BTC')) assert isinstance(dp.ohlcv('UNITTEST/BTC'), DataFrame) assert dp.ohlcv('UNITTEST/BTC') is not ticker_history assert dp.ohlcv('UNITTEST/BTC', copy=False) is ticker_history assert dp.ohlcv('NONESENSE/AAA') is None + default_conf['runmode'] = RunMode.LIVE + dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.LIVE + assert isinstance(dp.ohlcv('UNITTEST/BTC'), DataFrame) + + default_conf['runmode'] = RunMode.BACKTEST + dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.BACKTEST + assert dp.ohlcv('UNITTEST/BTC') is None + def test_historic_ohlcv(mocker, default_conf, ticker_history): From 646e98da55bd0e20c2a6dfa42e5a5272d319bead Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Dec 2018 13:00:50 +0100 Subject: [PATCH 20/49] Always return dataframe --- docs/bot-optimization.md | 2 +- freqtrade/data/dataprovider.py | 4 ++-- freqtrade/exchange/__init__.py | 2 +- freqtrade/tests/data/test_dataprovider.py | 5 +++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 6d03f55ac..7fee559b8 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -247,7 +247,7 @@ All methods return `None` in case of failure (do not raise an exception). #### Possible options for DataProvider - `available_pairs` - Property containing cached pairs -- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for all pairs in the whitelist +- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for all pairs in the whitelist, returns DataFrame or empty DataFrame - `historic_ohlcv(pair, ticker_interval)` - Data stored on disk - `runmode` - Property containing the current runmode. diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index e75f6ce33..4f854e647 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -37,7 +37,7 @@ class DataProvider(object): """ return list(self._exchange._klines.keys()) - def ohlcv(self, pair: str, copy: bool = True) -> List[str]: + def ohlcv(self, pair: str, copy: bool = True) -> DataFrame: """ get ohlcv data for the given pair as DataFrame Please check `available_pairs` to verify which pairs are currently cached. @@ -49,7 +49,7 @@ class DataProvider(object): if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): return self._exchange.klines(pair, copy) else: - return None + return DataFrame() def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame: """ diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index e0e4d7723..03b28939e 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -162,7 +162,7 @@ class Exchange(object): if pair in self._klines: return self._klines[pair].copy() if copy else self._klines[pair] else: - return None + return DataFrame() def set_sandbox(self, api, exchange_config: dict, name: str): if exchange_config.get('sandbox'): diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py index 9988d0fef..9f17da391 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/freqtrade/tests/data/test_dataprovider.py @@ -18,7 +18,8 @@ def test_ohlcv(mocker, default_conf, ticker_history): assert isinstance(dp.ohlcv('UNITTEST/BTC'), DataFrame) assert dp.ohlcv('UNITTEST/BTC') is not ticker_history assert dp.ohlcv('UNITTEST/BTC', copy=False) is ticker_history - assert dp.ohlcv('NONESENSE/AAA') is None + assert not dp.ohlcv('UNITTEST/BTC').empty + assert dp.ohlcv('NONESENSE/AAA').empty default_conf['runmode'] = RunMode.LIVE dp = DataProvider(default_conf, exchange) @@ -28,7 +29,7 @@ def test_ohlcv(mocker, default_conf, ticker_history): default_conf['runmode'] = RunMode.BACKTEST dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.BACKTEST - assert dp.ohlcv('UNITTEST/BTC') is None + assert dp.ohlcv('UNITTEST/BTC').empty def test_historic_ohlcv(mocker, default_conf, ticker_history): From 06ec1060797f86d84dd6afdca8da42c5b5b50705 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Dec 2018 13:07:22 +0100 Subject: [PATCH 21/49] simplify refresh_tickers --- freqtrade/exchange/__init__.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 03b28939e..84f20a7fe 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -530,30 +530,26 @@ class Exchange(object): logger.info("downloaded %s with length %s.", pair, len(data)) return data - def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> None: + def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> List[Tuple[str, List]]: """ - Refresh tickers asyncronously and set `_klines` of this object with the result + Refresh ohlcv asyncronously and set `_klines` with the result """ logger.debug("Refreshing klines for %d pairs", len(pair_list)) - asyncio.get_event_loop().run_until_complete( - self.async_get_candles_history(pair_list, ticker_interval)) - async def async_get_candles_history(self, pairs: List[str], - tick_interval: str) -> List[Tuple[str, List]]: - """Download ohlcv history for pair-list asyncronously """ # Calculating ticker interval in second - interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 + interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 input_coroutines = [] # Gather corotines to run - for pair in pairs: + for pair in pair_list: if not (self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= arrow.utcnow().timestamp and pair in self._klines): - input_coroutines.append(self._async_get_candle_history(pair, tick_interval)) + input_coroutines.append(self._async_get_candle_history(pair, ticker_interval)) else: logger.debug("Using cached klines data for %s ...", pair) - tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + tickers = asyncio.get_event_loop().run_until_complete( + asyncio.gather(*input_coroutines, return_exceptions=True)) # handle caching for res in tickers: From a206777fe50c2ad25c71cdf14c0705bc12d30f0d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Dec 2018 13:13:07 +0100 Subject: [PATCH 22/49] Rename refresh_tickers to refresh_latest_ohlcv --- freqtrade/data/dataprovider.py | 2 +- freqtrade/exchange/__init__.py | 5 +++-- freqtrade/freqtradebot.py | 4 ++-- freqtrade/optimize/backtesting.py | 2 +- freqtrade/tests/exchange/test_exchange.py | 6 +++--- freqtrade/tests/optimize/test_backtesting.py | 4 ++-- freqtrade/tests/test_freqtradebot.py | 2 +- scripts/plot_dataframe.py | 2 +- 8 files changed, 14 insertions(+), 13 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 4f854e647..628ca1dd4 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -27,7 +27,7 @@ class DataProvider(object): """ Refresh data, called with each cycle """ - self._exchange.refresh_tickers(pairlist, self._config['ticker_interval']) + self._exchange.refresh_latest_ohlcv(pairlist, self._config['ticker_interval']) @property def available_pairs(self) -> List[str]: diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 84f20a7fe..03acc2e61 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -530,9 +530,10 @@ class Exchange(object): logger.info("downloaded %s with length %s.", pair, len(data)) return data - def refresh_tickers(self, pair_list: List[str], ticker_interval: str) -> List[Tuple[str, List]]: + def refresh_latest_ohlcv(self, pair_list: List[str], + ticker_interval: str) -> List[Tuple[str, List]]: """ - Refresh ohlcv asyncronously and set `_klines` with the result + Refresh in-memory ohlcv asyncronously and set `_klines` with the result """ logger.debug("Refreshing klines for %d pairs", len(pair_list)) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 071cbaf2e..41cd35da7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -158,9 +158,9 @@ class FreqtradeBot(object): self.active_pair_whitelist = self.pairlists.whitelist # Calculating Edge positiong - # Should be called before refresh_tickers + # Should be called before refresh_latest_ohlcv # Otherwise it will override cached klines in exchange - # with delta value (klines only from last refresh_pairs) + # with delta value (klines only from last refresh_latest_ohlcv) if self.edge: self.edge.calculate() self.active_pair_whitelist = self.edge.adjust(self.active_pair_whitelist) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 88ad4cc60..192f8cff0 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -368,7 +368,7 @@ class Backtesting(object): if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') - self.exchange.refresh_tickers(pairs, self.ticker_interval) + self.exchange.refresh_latest_ohlcv(pairs, self.ticker_interval) data = self.exchange._klines else: logger.info('Using local backtesting data (using whitelist in given config) ...') diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 26808e78a..e0be9a4bb 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -778,7 +778,7 @@ def test_get_history(default_conf, mocker, caplog): assert len(ret) == 2 -def test_refresh_tickers(mocker, default_conf, caplog) -> None: +def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: tick = [ [ (arrow.utcnow().timestamp - 1) * 1000, # unix timestamp ms @@ -805,7 +805,7 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None: pairs = ['IOTA/ETH', 'XRP/ETH'] # empty dicts assert not exchange._klines - exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') + exchange.refresh_latest_ohlcv(['IOTA/ETH', 'XRP/ETH'], '5m') assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples) assert exchange._klines @@ -822,7 +822,7 @@ def test_refresh_tickers(mocker, default_conf, caplog) -> None: assert exchange.klines(pair, copy=False) is exchange.klines(pair, copy=False) # test caching - exchange.refresh_tickers(['IOTA/ETH', 'XRP/ETH'], '5m') + exchange.refresh_latest_ohlcv(['IOTA/ETH', 'XRP/ETH'], '5m') assert exchange._api_async.fetch_ohlcv.call_count == 2 assert log_has(f"Using cached klines data for {pairs[0]} ...", caplog.record_tuples) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index beef1b16e..b0e5da0f8 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -448,7 +448,7 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.data.history.load_data', mocked_load_data) mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe) - mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', @@ -483,7 +483,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={})) mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe) - mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.optimize.backtesting.Backtesting', diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1149a69e9..ca6190ae4 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :return: None """ freqtrade.strategy.get_signal = lambda e, s, t: value - freqtrade.exchange.refresh_tickers = lambda p, i: None + freqtrade.exchange.refresh_latest_ohlcv = lambda p, i: None def patch_RPCManager(mocker) -> MagicMock: diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index ae9cd7f1d..8f3e8327a 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -138,7 +138,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None: tickers = {} if args.live: logger.info('Downloading pair.') - exchange.refresh_tickers([pair], tick_interval) + exchange.refresh_latest_ohlcv([pair], tick_interval) tickers[pair] = exchange.klines(pair) else: tickers = history.load_data( From b981cfcaa0b800cac89b8417690a42e96a90e13d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Dec 2018 14:15:13 +0100 Subject: [PATCH 23/49] remove comment which proves untrue now --- freqtrade/freqtradebot.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 41cd35da7..6b27130bc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -158,9 +158,6 @@ class FreqtradeBot(object): self.active_pair_whitelist = self.pairlists.whitelist # Calculating Edge positiong - # Should be called before refresh_latest_ohlcv - # Otherwise it will override cached klines in exchange - # with delta value (klines only from last refresh_latest_ohlcv) if self.edge: self.edge.calculate() self.active_pair_whitelist = self.edge.adjust(self.active_pair_whitelist) From e503d811bd1e33eff318cadd6866bff97cab0993 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Dec 2018 14:18:49 +0100 Subject: [PATCH 24/49] Change logmessages to match functions called --- freqtrade/exchange/__init__.py | 6 +++--- freqtrade/tests/exchange/test_exchange.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 03acc2e61..243badfcb 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -525,7 +525,7 @@ class Exchange(object): for p, ticker in tickers: if p == pair: data.extend(ticker) - # Sort data again after extending the result - above calls return in "async order" order + # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) logger.info("downloaded %s with length %s.", pair, len(data)) return data @@ -535,7 +535,7 @@ class Exchange(object): """ Refresh in-memory ohlcv asyncronously and set `_klines` with the result """ - logger.debug("Refreshing klines for %d pairs", len(pair_list)) + logger.debug("Refreshing ohlcv data for %d pairs", len(pair_list)) # Calculating ticker interval in second interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 @@ -547,7 +547,7 @@ class Exchange(object): arrow.utcnow().timestamp and pair in self._klines): input_coroutines.append(self._async_get_candle_history(pair, ticker_interval)) else: - logger.debug("Using cached klines data for %s ...", pair) + logger.debug("Using cached ohlcv data for %s ...", pair) tickers = asyncio.get_event_loop().run_until_complete( asyncio.gather(*input_coroutines, return_exceptions=True)) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index e0be9a4bb..ee53f5cdd 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -807,7 +807,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: assert not exchange._klines exchange.refresh_latest_ohlcv(['IOTA/ETH', 'XRP/ETH'], '5m') - assert log_has(f'Refreshing klines for {len(pairs)} pairs', caplog.record_tuples) + assert log_has(f'Refreshing ohlcv data for {len(pairs)} pairs', caplog.record_tuples) assert exchange._klines assert exchange._api_async.fetch_ohlcv.call_count == 2 for pair in pairs: @@ -825,7 +825,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: exchange.refresh_latest_ohlcv(['IOTA/ETH', 'XRP/ETH'], '5m') assert exchange._api_async.fetch_ohlcv.call_count == 2 - assert log_has(f"Using cached klines data for {pairs[0]} ...", caplog.record_tuples) + assert log_has(f"Using cached ohlcv data for {pairs[0]} ...", caplog.record_tuples) @pytest.mark.asyncio @@ -854,7 +854,7 @@ async def test__async_get_candle_history(default_conf, mocker, caplog): assert res[0] == pair assert res[1] == tick assert exchange._api_async.fetch_ohlcv.call_count == 1 - assert not log_has(f"Using cached klines data for {pair} ...", caplog.record_tuples) + assert not log_has(f"Using cached ohlcv data for {pair} ...", caplog.record_tuples) # exchange = Exchange(default_conf) await async_ccxt_exception(mocker, default_conf, MagicMock(), From d6df3e55c05ea5f8a7e9950f86d5d4160d83a042 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Dec 2018 14:32:24 +0100 Subject: [PATCH 25/49] Return ticker_interval from async routine used to identify calls in refresh_latest_ohlcv --- freqtrade/exchange/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 243badfcb..7ce4a38c1 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -522,7 +522,7 @@ class Exchange(object): # Combine tickers data: List = [] - for p, ticker in tickers: + for p, ticker_interval, ticker in tickers: if p == pair: data.extend(ticker) # Sort data again after extending the result - above calls return in "async order" @@ -558,7 +558,8 @@ class Exchange(object): logger.warning("Async code raised an exception: %s", res.__class__.__name__) continue pair = res[0] - ticks = res[1] + tick_interval[1] + ticks = res[2] # keeping last candle time as last refreshed time of the pair if ticks: self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000 @@ -568,7 +569,11 @@ class Exchange(object): @retrier_async async def _async_get_candle_history(self, pair: str, tick_interval: str, - since_ms: Optional[int] = None) -> Tuple[str, List]: + since_ms: Optional[int] = None) -> Tuple[str, str, List]: + """ + Asyncronously gets candle histories using fetch_ohlcv + returns tuple: (pair, tick_interval, ohlcv_list) + """ try: # fetch ohlcv asynchronously logger.debug("fetching %s since %s ...", pair, since_ms) @@ -587,7 +592,7 @@ class Exchange(object): logger.exception("Error loading %s. Result was %s.", pair, data) return pair, [] logger.debug("done fetching %s ...", pair) - return pair, data + return pair, tick_interval, data except ccxt.NotSupported as e: raise OperationalException( From 5f61da30ed8049b2503de6bab402e8af8d607cd9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Dec 2018 19:30:47 +0100 Subject: [PATCH 26/49] Adjust tests to 3tuple return value from async method --- freqtrade/exchange/__init__.py | 1 + freqtrade/tests/exchange/test_exchange.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 7ce4a38c1..9a426c805 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -518,6 +518,7 @@ class Exchange(object): input_coroutines = [self._async_get_candle_history( pair, tick_interval, since) for since in range(since_ms, arrow.utcnow().timestamp * 1000, one_call)] + tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) # Combine tickers diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index ee53f5cdd..eaf48fa56 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -765,7 +765,7 @@ def test_get_history(default_conf, mocker, caplog): pair = 'ETH/BTC' async def mock_candle_hist(pair, tick_interval, since_ms): - return pair, tick + return pair, tick_interval, tick exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls @@ -850,9 +850,10 @@ async def test__async_get_candle_history(default_conf, mocker, caplog): pair = 'ETH/BTC' res = await exchange._async_get_candle_history(pair, "5m") assert type(res) is tuple - assert len(res) == 2 + assert len(res) == 3 assert res[0] == pair - assert res[1] == tick + assert res[1] == "5m" + assert res[2] == tick assert exchange._api_async.fetch_ohlcv.call_count == 1 assert not log_has(f"Using cached ohlcv data for {pair} ...", caplog.record_tuples) @@ -883,9 +884,10 @@ async def test__async_get_candle_history_empty(default_conf, mocker, caplog): pair = 'ETH/BTC' res = await exchange._async_get_candle_history(pair, "5m") assert type(res) is tuple - assert len(res) == 2 + assert len(res) == 3 assert res[0] == pair - assert res[1] == tick + assert res[1] == "5m" + assert res[2] == tick assert exchange._api_async.fetch_ohlcv.call_count == 1 @@ -1010,7 +1012,7 @@ async def test___async_get_candle_history_sort(default_conf, mocker): # Test the ticker history sort res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert res[0] == 'ETH/BTC' - ticks = res[1] + ticks = res[2] assert sort_mock.call_count == 1 assert ticks[0][0] == 1527830400000 @@ -1047,7 +1049,8 @@ async def test___async_get_candle_history_sort(default_conf, mocker): # Test the ticker history sort res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert res[0] == 'ETH/BTC' - ticks = res[1] + assert res[1] == default_conf['ticker_interval'] + ticks = res[2] # Sorted not called again - data is already in order assert sort_mock.call_count == 0 assert ticks[0][0] == 1527827700000 From 0aa0b1d4feeb7ad3abe366d760852db879bbcd3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Dec 2018 07:15:21 +0100 Subject: [PATCH 27/49] Store tickers by pair / ticker_interval --- freqtrade/data/dataprovider.py | 18 ++++++++++++----- freqtrade/exchange/__init__.py | 32 ++++++++++++++++--------------- freqtrade/freqtradebot.py | 14 ++++++++++---- freqtrade/optimize/backtesting.py | 5 +++-- 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 628ca1dd4..9b9c73e55 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -6,7 +6,7 @@ Common Interface for bot and strategy to access data. """ import logging from pathlib import Path -from typing import List +from typing import List, Tuple from pandas import DataFrame @@ -23,11 +23,11 @@ class DataProvider(object): self._config = config self._exchange = exchange - def refresh(self, pairlist: List[str]) -> None: + def refresh(self, pairlist: List[Tuple[str, str]]) -> None: """ Refresh data, called with each cycle """ - self._exchange.refresh_latest_ohlcv(pairlist, self._config['ticker_interval']) + self._exchange.refresh_latest_ohlcv(pairlist) @property def available_pairs(self) -> List[str]: @@ -37,23 +37,31 @@ class DataProvider(object): """ return list(self._exchange._klines.keys()) - def ohlcv(self, pair: str, copy: bool = True) -> DataFrame: + def ohlcv(self, pair: str, tick_interval: str = None, copy: bool = True) -> DataFrame: """ get ohlcv data for the given pair as DataFrame Please check `available_pairs` to verify which pairs are currently cached. :param pair: pair to get the data for + :param tick_interval: ticker_interval to get pair for :param copy: copy dataframe before returning. Use false only for RO operations (where the dataframe is not modified) """ # TODO: Should not be stored in exchange but in this class if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - return self._exchange.klines(pair, copy) + if tick_interval: + pairtick = (pair, tick_interval) + else: + pairtick = (pair, self._config['ticker_interval']) + + return self._exchange.klines(pairtick, copy=copy) else: return DataFrame() def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame: """ get historic ohlcv data stored for backtesting + :param pair: pair to get the data for + :param tick_interval: ticker_interval to get pair for """ return load_pair_history(pair=pair, ticker_interval=ticker_interval, diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 9a426c805..21eab39d4 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -83,7 +83,7 @@ class Exchange(object): self._pairs_last_refresh_time: Dict[str, int] = {} # Holds candles - self._klines: Dict[str, DataFrame] = {} + self._klines: Dict[Tuple[str, str], DataFrame] = {} # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} @@ -158,9 +158,10 @@ class Exchange(object): """exchange ccxt id""" return self._api.id - def klines(self, pair: str, copy=True) -> DataFrame: - if pair in self._klines: - return self._klines[pair].copy() if copy else self._klines[pair] + def klines(self, pair_interval: Tuple[str, str], copy=True) -> DataFrame: + # create key tuple + if pair_interval in self._klines: + return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] else: return DataFrame() @@ -531,24 +532,24 @@ class Exchange(object): logger.info("downloaded %s with length %s.", pair, len(data)) return data - def refresh_latest_ohlcv(self, pair_list: List[str], - ticker_interval: str) -> List[Tuple[str, List]]: + def refresh_latest_ohlcv(self, pair_list: List[Tuple[str, str]]) -> List[Tuple[str, List]]: """ - Refresh in-memory ohlcv asyncronously and set `_klines` with the result + Refresh in-memory ohlcv asyncronously and set `_klines` with the result """ logger.debug("Refreshing ohlcv data for %d pairs", len(pair_list)) - # Calculating ticker interval in second - interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 input_coroutines = [] # Gather corotines to run - for pair in pair_list: + for pair, ticker_interval in pair_list: + # Calculating ticker interval in second + interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 + if not (self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= - arrow.utcnow().timestamp and pair in self._klines): + arrow.utcnow().timestamp and (pair, ticker_interval) in self._klines): input_coroutines.append(self._async_get_candle_history(pair, ticker_interval)) else: - logger.debug("Using cached ohlcv data for %s ...", pair) + logger.debug("Using cached ohlcv data for %s, %s ...", pair, ticker_interval) tickers = asyncio.get_event_loop().run_until_complete( asyncio.gather(*input_coroutines, return_exceptions=True)) @@ -559,13 +560,14 @@ class Exchange(object): logger.warning("Async code raised an exception: %s", res.__class__.__name__) continue pair = res[0] - tick_interval[1] + tick_interval = res[1] ticks = res[2] # keeping last candle time as last refreshed time of the pair if ticks: self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000 - # keeping parsed dataframe in cache - self._klines[pair] = parse_ticker_dataframe(ticks, tick_interval, fill_missing=True) + # keeping parsed dataframe in cache + self._klines[(pair, tick_interval)] = parse_ticker_dataframe( + ticks, tick_interval, fill_missing=True) return tickers @retrier_async diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6b27130bc..cc75fd89b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -170,8 +170,11 @@ class FreqtradeBot(object): self.active_pair_whitelist.extend([trade.pair for trade in trades if trade.pair not in self.active_pair_whitelist]) + # Create pair-whitelist tuple with (pair, ticker_interval) + pair_whitelist = [(pair, self.config['ticker_interval']) + for pair in self.active_pair_whitelist] # Refreshing candles - self.dataprovider.refresh(self.active_pair_whitelist) + self.dataprovider.refresh(pair_whitelist) # First process current opened trades for trade in trades: @@ -321,7 +324,9 @@ class FreqtradeBot(object): # running get_signal on historical data fetched for _pair in whitelist: - (buy, sell) = self.strategy.get_signal(_pair, interval, self.dataprovider.ohlcv(_pair)) + (buy, sell) = self.strategy.get_signal( + _pair, interval, self.dataprovider.ohlcv(_pair, self.strategy.ticker_interval)) + if buy and not sell: stake_amount = self._get_trade_stake_amount(_pair) if not stake_amount: @@ -582,8 +587,9 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, - self.dataprovider.ohlcv(trade.pair)) + (buy, sell) = self.strategy.get_signal( + trade.pair, self.strategy.ticker_interval, + self.dataprovider.ohlcv(trade.pair, self.strategy.ticker_interval)) config_ask_strategy = self.config.get('ask_strategy', {}) if config_ask_strategy.get('use_order_book', False): diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 192f8cff0..a3e74704b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -368,8 +368,9 @@ class Backtesting(object): if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') - self.exchange.refresh_latest_ohlcv(pairs, self.ticker_interval) - data = self.exchange._klines + self.exchange.refresh_latest_ohlcv([(pair, self.ticker_interval) for pair in pairs]) + data = {key[0]: value for key, value in self.exchange._klines.items()} + else: logger.info('Using local backtesting data (using whitelist in given config) ...') From a9abc25785fe865c6e9f6a7febabcdc1b49f25fb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Dec 2018 07:15:49 +0100 Subject: [PATCH 28/49] Improve data-provider tests --- docs/bot-optimization.md | 4 +-- freqtrade/tests/data/test_dataprovider.py | 30 ++++++++++++++--------- freqtrade/tests/exchange/test_exchange.py | 9 ++++--- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 7fee559b8..979adcd08 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -235,11 +235,11 @@ if self.dp: if dp.runmode == 'live': if 'ETH/BTC' in self.dp.available_pairs: data_eth = self.dp.ohlcv(pair='ETH/BTC', - ticker_interval=ticker_interval) + ticker_interval=ticker_interval) else: # Get historic ohlcv data (cached on disk). history_eth = self.dp.historic_ohlcv(pair='ETH/BTC', - ticker_interval='1h') + ticker_interval='1h') ``` All methods return `None` in case of failure (do not raise an exception). diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py index 9f17da391..89b709572 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/freqtrade/tests/data/test_dataprovider.py @@ -9,27 +9,31 @@ from freqtrade.tests.conftest import get_patched_exchange def test_ohlcv(mocker, default_conf, ticker_history): default_conf['runmode'] = RunMode.DRY_RUN + tick_interval = default_conf['ticker_interval'] exchange = get_patched_exchange(mocker, default_conf) - exchange._klines['XRP/BTC'] = ticker_history - exchange._klines['UNITTEST/BTC'] = ticker_history + exchange._klines[('XRP/BTC', tick_interval)] = ticker_history + exchange._klines[('UNITTEST/BTC', tick_interval)] = ticker_history dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN - assert ticker_history.equals(dp.ohlcv('UNITTEST/BTC')) - assert isinstance(dp.ohlcv('UNITTEST/BTC'), DataFrame) - assert dp.ohlcv('UNITTEST/BTC') is not ticker_history - assert dp.ohlcv('UNITTEST/BTC', copy=False) is ticker_history - assert not dp.ohlcv('UNITTEST/BTC').empty - assert dp.ohlcv('NONESENSE/AAA').empty + assert ticker_history.equals(dp.ohlcv('UNITTEST/BTC', tick_interval)) + assert isinstance(dp.ohlcv('UNITTEST/BTC', tick_interval), DataFrame) + assert dp.ohlcv('UNITTEST/BTC', tick_interval) is not ticker_history + assert dp.ohlcv('UNITTEST/BTC', tick_interval, copy=False) is ticker_history + assert not dp.ohlcv('UNITTEST/BTC', tick_interval).empty + assert dp.ohlcv('NONESENSE/AAA', tick_interval).empty + + # Test with and without parameter + assert dp.ohlcv('UNITTEST/BTC', tick_interval).equals(dp.ohlcv('UNITTEST/BTC')) default_conf['runmode'] = RunMode.LIVE dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.LIVE - assert isinstance(dp.ohlcv('UNITTEST/BTC'), DataFrame) + assert isinstance(dp.ohlcv('UNITTEST/BTC', tick_interval), DataFrame) default_conf['runmode'] = RunMode.BACKTEST dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.BACKTEST - assert dp.ohlcv('UNITTEST/BTC').empty + assert dp.ohlcv('UNITTEST/BTC', tick_interval).empty def test_historic_ohlcv(mocker, default_conf, ticker_history): @@ -49,8 +53,10 @@ def test_historic_ohlcv(mocker, default_conf, ticker_history): def test_available_pairs(mocker, default_conf, ticker_history): exchange = get_patched_exchange(mocker, default_conf) - exchange._klines['XRP/BTC'] = ticker_history - exchange._klines['UNITTEST/BTC'] = ticker_history + + tick_interval = default_conf['ticker_interval'] + exchange._klines[('XRP/BTC', tick_interval)] = ticker_history + exchange._klines[('UNITTEST/BTC', tick_interval)] = ticker_history dp = DataProvider(default_conf, exchange) assert len(dp.available_pairs) == 2 diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index eaf48fa56..8fc3f1a32 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -802,10 +802,10 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: exchange = get_patched_exchange(mocker, default_conf) exchange._api_async.fetch_ohlcv = get_mock_coro(tick) - pairs = ['IOTA/ETH', 'XRP/ETH'] + pairs = [('IOTA/ETH', '5m'), ('XRP/ETH', '5m')] # empty dicts assert not exchange._klines - exchange.refresh_latest_ohlcv(['IOTA/ETH', 'XRP/ETH'], '5m') + exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m')]) assert log_has(f'Refreshing ohlcv data for {len(pairs)} pairs', caplog.record_tuples) assert exchange._klines @@ -822,10 +822,11 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: assert exchange.klines(pair, copy=False) is exchange.klines(pair, copy=False) # test caching - exchange.refresh_latest_ohlcv(['IOTA/ETH', 'XRP/ETH'], '5m') + exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m')]) assert exchange._api_async.fetch_ohlcv.call_count == 2 - assert log_has(f"Using cached ohlcv data for {pairs[0]} ...", caplog.record_tuples) + assert log_has(f"Using cached ohlcv data for {pairs[0][0]}, {pairs[0][1]} ...", + caplog.record_tuples) @pytest.mark.asyncio From f0af4601f9820f29b8230101d7a6fb456c7017e4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Dec 2018 07:20:03 +0100 Subject: [PATCH 29/49] Adopt plot_dataframe to work with --live --- freqtrade/data/dataprovider.py | 2 +- freqtrade/exchange/__init__.py | 2 +- freqtrade/tests/test_freqtradebot.py | 2 +- scripts/plot_dataframe.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 9b9c73e55..c6f9c9c88 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -32,7 +32,7 @@ class DataProvider(object): @property def available_pairs(self) -> List[str]: """ - Return a list of pairs for which data is currently cached. + Return a list of tuples containing pair, tick_interval for which data is currently cached. Should be whitelist + open trades. """ return list(self._exchange._klines.keys()) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 21eab39d4..1fff1c1b5 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -579,7 +579,7 @@ class Exchange(object): """ try: # fetch ohlcv asynchronously - logger.debug("fetching %s since %s ...", pair, since_ms) + logger.debug("fetching %s, %s since %s ...", pair, tick_interval, since_ms) data = await self._api_async.fetch_ohlcv(pair, timeframe=tick_interval, since=since_ms) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index ca6190ae4..ef2815bed 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -43,7 +43,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :return: None """ freqtrade.strategy.get_signal = lambda e, s, t: value - freqtrade.exchange.refresh_latest_ohlcv = lambda p, i: None + freqtrade.exchange.refresh_latest_ohlcv = lambda p: None def patch_RPCManager(mocker) -> MagicMock: diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 8f3e8327a..8c0793174 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -138,8 +138,8 @@ def plot_analyzed_dataframe(args: Namespace) -> None: tickers = {} if args.live: logger.info('Downloading pair.') - exchange.refresh_latest_ohlcv([pair], tick_interval) - tickers[pair] = exchange.klines(pair) + exchange.refresh_latest_ohlcv([(pair, tick_interval)]) + tickers[pair] = exchange.klines((pair, tick_interval)) else: tickers = history.load_data( datadir=Path(_CONF.get("datadir")), From 6525a838d1dea85530d03d89e09d4451ead0aed7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Dec 2018 10:25:47 +0100 Subject: [PATCH 30/49] Adjust documentation to tuple use --- docs/bot-optimization.md | 26 ++++++++++++++++++-------- freqtrade/data/dataprovider.py | 1 - 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 979adcd08..51e2f0f63 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -228,12 +228,23 @@ The strategy provides access to the `DataProvider`. This allows you to get addit **NOTE**: The DataProvier is currently not available during backtesting / hyperopt. +All methods return `None` in case of failure (do not raise an exception). + Please always check if the `DataProvider` is available to avoid failures during backtesting. +#### Possible options for DataProvider + +- `available_pairs` - Property with tuples listing cached pairs with their intervals. (pair, interval) +- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for all pairs in the whitelist, returns DataFrame or empty DataFrame +- `historic_ohlcv(pair, ticker_interval)` - Data stored on disk +- `runmode` - Property containing the current runmode. + +#### ohlcv / historic_ohlcv + ``` python if self.dp: if dp.runmode == 'live': - if 'ETH/BTC' in self.dp.available_pairs: + if ('ETH/BTC', ticker_interval) in self.dp.available_pairs: data_eth = self.dp.ohlcv(pair='ETH/BTC', ticker_interval=ticker_interval) else: @@ -242,14 +253,13 @@ if self.dp: ticker_interval='1h') ``` -All methods return `None` in case of failure (do not raise an exception). +#### Available Pairs -#### Possible options for DataProvider - -- `available_pairs` - Property containing cached pairs -- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for all pairs in the whitelist, returns DataFrame or empty DataFrame -- `historic_ohlcv(pair, ticker_interval)` - Data stored on disk -- `runmode` - Property containing the current runmode. +``` python +if self.dp: + for pair, ticker in self.dp.available_pairs: + print(f"available {pair}, {ticker}") +``` ### Additional data - Wallets diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index c6f9c9c88..fdb3714fc 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -46,7 +46,6 @@ class DataProvider(object): :param copy: copy dataframe before returning. Use false only for RO operations (where the dataframe is not modified) """ - # TODO: Should not be stored in exchange but in this class if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): if tick_interval: pairtick = (pair, tick_interval) From d7df5d57153c41d5c3a292b36bb312da6242afcb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Jan 2019 06:44:08 +0100 Subject: [PATCH 31/49] Keep last_pair_refresh as tuple asw ell --- freqtrade/exchange/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 1fff1c1b5..44642476c 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -80,7 +80,7 @@ class Exchange(object): self._cached_ticker: Dict[str, Any] = {} # Holds last candle refreshed time of each pair - self._pairs_last_refresh_time: Dict[str, int] = {} + self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {} # Holds candles self._klines: Dict[Tuple[str, str], DataFrame] = {} @@ -545,7 +545,7 @@ class Exchange(object): # Calculating ticker interval in second interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 - if not (self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= + if not (self._pairs_last_refresh_time.get((pair, ticker_interval), 0) + interval_in_sec >= arrow.utcnow().timestamp and (pair, ticker_interval) in self._klines): input_coroutines.append(self._async_get_candle_history(pair, ticker_interval)) else: @@ -564,7 +564,7 @@ class Exchange(object): ticks = res[2] # keeping last candle time as last refreshed time of the pair if ticks: - self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000 + self._pairs_last_refresh_time[(pair, tick_interval)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache self._klines[(pair, tick_interval)] = parse_ticker_dataframe( ticks, tick_interval, fill_missing=True) From 1e749a0f9b9b28f0862d57db0b825905e9c1a713 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Jan 2019 19:25:39 +0100 Subject: [PATCH 32/49] Rename variable to be clearer --- freqtrade/freqtradebot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index cc75fd89b..74d2cc43d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -171,10 +171,10 @@ class FreqtradeBot(object): if trade.pair not in self.active_pair_whitelist]) # Create pair-whitelist tuple with (pair, ticker_interval) - pair_whitelist = [(pair, self.config['ticker_interval']) - for pair in self.active_pair_whitelist] + pair_whitelist_tuple = [(pair, self.config['ticker_interval']) + for pair in self.active_pair_whitelist] # Refreshing candles - self.dataprovider.refresh(pair_whitelist) + self.dataprovider.refresh(pair_whitelist_tuple) # First process current opened trades for trade in trades: From a2bc1da669a3f7fbfb0d401ecd5b7a2739cc710a Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Jan 2019 19:28:53 +0100 Subject: [PATCH 33/49] Remove private var from class instance it's overwritten in __init__ anyway --- freqtrade/strategy/interface.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 7210f5c78..ecb32b2b5 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -95,9 +95,6 @@ class IStrategy(ABC): # run "populate_indicators" only for new candle process_only_new_candles: bool = False - # Dict to determine if analysis is necessary - _last_candle_seen_per_pair: Dict[str, datetime] = {} - # Class level variables (intentional) containing # the dataprovider (dp) (access to other candles, historic data, ...) # and wallets - access to the current balance. @@ -106,7 +103,8 @@ class IStrategy(ABC): def __init__(self, config: dict) -> None: self.config = config - self._last_candle_seen_per_pair = {} + # Dict to determine if analysis is necessary + self._last_candle_seen_per_pair: Dict[str, datetime] = {} @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: From e7800aa88a1a559f0973531628fd898b5b882f43 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Jan 2019 19:30:59 +0100 Subject: [PATCH 34/49] Import only what's necessary --- freqtrade/resolvers/strategy_resolver.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 0d1c9f9a0..0f71501ae 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -3,11 +3,11 @@ """ This module load custom strategies """ -import inspect import logging import tempfile from base64 import urlsafe_b64decode from collections import OrderedDict +from inspect import getfullargspec from pathlib import Path from typing import Dict, Optional @@ -126,11 +126,9 @@ class StrategyResolver(IResolver): if strategy: logger.info('Using resolved strategy %s from \'%s\'', strategy_name, _path) strategy._populate_fun_len = len( - inspect.getfullargspec(strategy.populate_indicators).args) - strategy._buy_fun_len = len( - inspect.getfullargspec(strategy.populate_buy_trend).args) - strategy._sell_fun_len = len( - inspect.getfullargspec(strategy.populate_sell_trend).args) + getfullargspec(strategy.populate_indicators).args) + strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) + strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args) return import_strategy(strategy, config=config) except FileNotFoundError: From 27b2021726cedb53f24b7beeee9d599fa630f9f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Jan 2019 19:56:15 +0100 Subject: [PATCH 35/49] Only run once per pair --- freqtrade/exchange/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 44642476c..62730e034 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -541,7 +541,7 @@ class Exchange(object): input_coroutines = [] # Gather corotines to run - for pair, ticker_interval in pair_list: + for pair, ticker_interval in set(pair_list): # Calculating ticker interval in second interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 From 7b138ef3b496b33e4c27790eec42b394f589172e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Jan 2019 20:03:19 +0100 Subject: [PATCH 36/49] Add warning about strategy/backtesting --- docs/bot-optimization.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 51e2f0f63..cefcacee6 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -47,6 +47,12 @@ python3 ./freqtrade/main.py --strategy AwesomeStrategy **For the following section we will use the [user_data/strategies/test_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/test_strategy.py) file as reference.** +!!! Note: Strategies and Backtesting + To avoid problems and unexpected differences between Backtesting and dry/live modes, please be aware + that during backtesting the full time-interval is passed to the `populate_*()` methods at once. + It is therefore best to use vectorized operations (across the whole dataframe, not loops) and + avoid index referencing (`df.iloc[-1]`), but instead use `df.shift()` to get to the previous candle. + ### Customize Indicators Buy and sell strategies need indicators. You can add more indicators by extending the list contained in the method `populate_indicators()` from your strategy file. From d6cdfc58afcd5345ebdef573e1683ea29e3146f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Jan 2019 20:19:34 +0100 Subject: [PATCH 37/49] Fix mypy hickup after changing list to tuples --- freqtrade/data/dataprovider.py | 2 +- freqtrade/tests/data/test_dataprovider.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index fdb3714fc..ce1b480f9 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -30,7 +30,7 @@ class DataProvider(object): self._exchange.refresh_latest_ohlcv(pairlist) @property - def available_pairs(self) -> List[str]: + def available_pairs(self) -> List[Tuple[str, str]]: """ Return a list of tuples containing pair, tick_interval for which data is currently cached. Should be whitelist + open trades. diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py index 89b709572..834ff94b5 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/freqtrade/tests/data/test_dataprovider.py @@ -60,4 +60,4 @@ def test_available_pairs(mocker, default_conf, ticker_history): dp = DataProvider(default_conf, exchange) assert len(dp.available_pairs) == 2 - assert dp.available_pairs == ['XRP/BTC', 'UNITTEST/BTC'] + assert dp.available_pairs == [('XRP/BTC', tick_interval), ('UNITTEST/BTC', tick_interval)] From 6e2de75bcbfb755097477821818db2c49f0e91b7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Jan 2019 20:22:27 +0100 Subject: [PATCH 38/49] Add additional_pairs to strategy --- freqtrade/strategy/interface.py | 13 +++++++++++++ user_data/strategies/test_strategy.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index ecb32b2b5..c7ec8dda6 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -133,6 +133,19 @@ class IStrategy(ABC): :return: DataFrame with sell column """ + def additional_pairs(self) -> List[Tuple[str, str]]: + """ + Define additional pair/interval combinations to be cached from the exchange. + These pair/interval combinations are non-tradeable, unless they are part + of the whitelist as well. + For more information, please consult the documentation + :return: List of tuples in the format (pair, interval) + Sample: return [("ETH/USDT", "5m"), + ("BTC/USDT", "15m"), + ] + """ + return [] + def get_strategy_name(self) -> str: """ Returns strategy class name diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 048b398c7..314787f7a 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -67,6 +67,19 @@ class TestStrategy(IStrategy): 'sell': 'gtc' } + def additional_pairs(self): + """ + Define additional pair/interval combinations to be cached from the exchange. + These pair/interval combinations are non-tradeable, unless they are part + of the whitelist as well. + For more information, please consult the documentation + :return: List of tuples in the format (pair, interval) + Sample: return [("ETH/USDT", "5m"), + ("BTC/USDT", "15m"), + ] + """ + return [] + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame From fc92491a4780873dded3ee3a5ae36a540d95afd2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Jan 2019 20:22:49 +0100 Subject: [PATCH 39/49] Add documentation for additional_pairs --- docs/bot-optimization.md | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index cefcacee6..9200d6b1b 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -232,7 +232,8 @@ Currently this is `pair`, which can be accessed using `metadata['pair']` - and w The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. -**NOTE**: The DataProvier is currently not available during backtesting / hyperopt. +!!!Note: + The DataProvier is currently not available during backtesting / hyperopt, but this is planned for the future. All methods return `None` in case of failure (do not raise an exception). @@ -259,6 +260,10 @@ if self.dp: ticker_interval='1h') ``` +!!! Warning: Warning about backtesting + Be carefull when using dataprovider in backtesting. `historic_ohlcv()` provides the full time-range in one go, + so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). + #### Available Pairs ``` python @@ -267,11 +272,35 @@ if self.dp: print(f"available {pair}, {ticker}") ``` +#### Get data for non-tradeable pairs + +Data for additional pairs (reference pairs) can be beneficial for some strategies. +Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see above). +These parts will **not** be traded unless they are also specified in the pair whitelist, or have been selected by Dynamic Whitelisting. + +The pairs need to be specified as tuples in the format `("pair", "interval")`, with pair as the first and time interval as the second argument. + +Sample: + +``` python +def additional_pairs(self): + return [("ETH/USDT", "5m"), + ("BTC/TUSD", "15m"), + ] +``` + +!!! Warning: + As these pairs will be refreshed as part of the regular whitelist refresh, it's best to keep this list short. + All intervals and all pairs can be specified as long as they are available (and active) on the used exchange. + It is however better to use resampling to longer time-intervals when possible + to avoid hammering the exchange with too many requests and risk beeing blocked. + ### Additional data - Wallets The strategy provides access to the `Wallets` object. This contains the current balances on the exchange. -**NOTE**: Wallets is not available during backtesting / hyperopt. +!!!NOTE: + Wallets is not available during backtesting / hyperopt. Please always check if `Wallets` is available to avoid failures during backtesting. From e66808bb02cb3e697033252c8261ebb168d99ab0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 22 Jan 2019 06:55:40 +0100 Subject: [PATCH 40/49] Add additional pairs to refresh call --- freqtrade/data/dataprovider.py | 9 +++++-- freqtrade/freqtradebot.py | 3 ++- freqtrade/tests/data/test_dataprovider.py | 30 +++++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index ce1b480f9..f69b1287d 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -23,11 +23,16 @@ class DataProvider(object): self._config = config self._exchange = exchange - def refresh(self, pairlist: List[Tuple[str, str]]) -> None: + def refresh(self, + pairlist: List[Tuple[str, str]], + helping_pairs: List[Tuple[str, str]] = None) -> None: """ Refresh data, called with each cycle """ - self._exchange.refresh_latest_ohlcv(pairlist) + if helping_pairs: + self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs) + else: + self._exchange.refresh_latest_ohlcv(pairlist) @property def available_pairs(self) -> List[Tuple[str, str]]: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 74d2cc43d..22ccab8c0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -174,7 +174,8 @@ class FreqtradeBot(object): pair_whitelist_tuple = [(pair, self.config['ticker_interval']) for pair in self.active_pair_whitelist] # Refreshing candles - self.dataprovider.refresh(pair_whitelist_tuple) + self.dataprovider.refresh(pair_whitelist_tuple, + self.strategy.additional_pairs()) # First process current opened trades for trade in trades: diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py index 834ff94b5..0fed86a4e 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/freqtrade/tests/data/test_dataprovider.py @@ -61,3 +61,33 @@ def test_available_pairs(mocker, default_conf, ticker_history): assert len(dp.available_pairs) == 2 assert dp.available_pairs == [('XRP/BTC', tick_interval), ('UNITTEST/BTC', tick_interval)] + + +def test_refresh(mocker, default_conf, ticker_history): + refresh_mock = MagicMock() + mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', refresh_mock) + + exchange = get_patched_exchange(mocker, default_conf, id='binance') + tick_interval = default_conf['ticker_interval'] + pairs = [('XRP/BTC', tick_interval), + ('UNITTEST/BTC', tick_interval), + ] + + pairs_non_trad = [('ETH/USDT', tick_interval), + ('BTC/TUSD', "1h"), + ] + + dp = DataProvider(default_conf, exchange) + dp.refresh(pairs) + + assert refresh_mock.call_count == 1 + assert len(refresh_mock.call_args[0]) == 1 + assert len(refresh_mock.call_args[0][0]) == len(pairs) + assert refresh_mock.call_args[0][0] == pairs + + refresh_mock.reset_mock() + dp.refresh(pairs, pairs_non_trad) + assert refresh_mock.call_count == 1 + assert len(refresh_mock.call_args[0]) == 1 + assert len(refresh_mock.call_args[0][0]) == len(pairs) + len(pairs_non_trad) + assert refresh_mock.call_args[0][0] == pairs + pairs_non_trad From 1e7431a7b81df9087e0fbbfd2b6f2a0cf1646105 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 22 Jan 2019 06:56:54 +0100 Subject: [PATCH 41/49] Blackify --- freqtrade/tests/data/test_dataprovider.py | 65 +++++++++++------------ 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py index 0fed86a4e..b17bba273 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/freqtrade/tests/data/test_dataprovider.py @@ -8,74 +8,73 @@ from freqtrade.tests.conftest import get_patched_exchange def test_ohlcv(mocker, default_conf, ticker_history): - default_conf['runmode'] = RunMode.DRY_RUN - tick_interval = default_conf['ticker_interval'] + default_conf["runmode"] = RunMode.DRY_RUN + tick_interval = default_conf["ticker_interval"] exchange = get_patched_exchange(mocker, default_conf) - exchange._klines[('XRP/BTC', tick_interval)] = ticker_history - exchange._klines[('UNITTEST/BTC', tick_interval)] = ticker_history + exchange._klines[("XRP/BTC", tick_interval)] = ticker_history + exchange._klines[("UNITTEST/BTC", tick_interval)] = ticker_history dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN - assert ticker_history.equals(dp.ohlcv('UNITTEST/BTC', tick_interval)) - assert isinstance(dp.ohlcv('UNITTEST/BTC', tick_interval), DataFrame) - assert dp.ohlcv('UNITTEST/BTC', tick_interval) is not ticker_history - assert dp.ohlcv('UNITTEST/BTC', tick_interval, copy=False) is ticker_history - assert not dp.ohlcv('UNITTEST/BTC', tick_interval).empty - assert dp.ohlcv('NONESENSE/AAA', tick_interval).empty + assert ticker_history.equals(dp.ohlcv("UNITTEST/BTC", tick_interval)) + assert isinstance(dp.ohlcv("UNITTEST/BTC", tick_interval), DataFrame) + assert dp.ohlcv("UNITTEST/BTC", tick_interval) is not ticker_history + assert dp.ohlcv("UNITTEST/BTC", tick_interval, copy=False) is ticker_history + assert not dp.ohlcv("UNITTEST/BTC", tick_interval).empty + assert dp.ohlcv("NONESENSE/AAA", tick_interval).empty # Test with and without parameter - assert dp.ohlcv('UNITTEST/BTC', tick_interval).equals(dp.ohlcv('UNITTEST/BTC')) + assert dp.ohlcv("UNITTEST/BTC", tick_interval).equals(dp.ohlcv("UNITTEST/BTC")) - default_conf['runmode'] = RunMode.LIVE + default_conf["runmode"] = RunMode.LIVE dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.LIVE - assert isinstance(dp.ohlcv('UNITTEST/BTC', tick_interval), DataFrame) + assert isinstance(dp.ohlcv("UNITTEST/BTC", tick_interval), DataFrame) - default_conf['runmode'] = RunMode.BACKTEST + default_conf["runmode"] = RunMode.BACKTEST dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.BACKTEST - assert dp.ohlcv('UNITTEST/BTC', tick_interval).empty + assert dp.ohlcv("UNITTEST/BTC", tick_interval).empty def test_historic_ohlcv(mocker, default_conf, ticker_history): historymock = MagicMock(return_value=ticker_history) - mocker.patch('freqtrade.data.dataprovider.load_pair_history', historymock) + mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock) # exchange = get_patched_exchange(mocker, default_conf) dp = DataProvider(default_conf, None) - data = dp.historic_ohlcv('UNITTEST/BTC', "5m") + data = dp.historic_ohlcv("UNITTEST/BTC", "5m") assert isinstance(data, DataFrame) assert historymock.call_count == 1 - assert historymock.call_args_list[0][1]['datadir'] is None - assert historymock.call_args_list[0][1]['refresh_pairs'] is False - assert historymock.call_args_list[0][1]['ticker_interval'] == '5m' + assert historymock.call_args_list[0][1]["datadir"] is None + assert historymock.call_args_list[0][1]["refresh_pairs"] is False + assert historymock.call_args_list[0][1]["ticker_interval"] == "5m" def test_available_pairs(mocker, default_conf, ticker_history): exchange = get_patched_exchange(mocker, default_conf) - tick_interval = default_conf['ticker_interval'] - exchange._klines[('XRP/BTC', tick_interval)] = ticker_history - exchange._klines[('UNITTEST/BTC', tick_interval)] = ticker_history + tick_interval = default_conf["ticker_interval"] + exchange._klines[("XRP/BTC", tick_interval)] = ticker_history + exchange._klines[("UNITTEST/BTC", tick_interval)] = ticker_history dp = DataProvider(default_conf, exchange) assert len(dp.available_pairs) == 2 - assert dp.available_pairs == [('XRP/BTC', tick_interval), ('UNITTEST/BTC', tick_interval)] + assert dp.available_pairs == [ + ("XRP/BTC", tick_interval), + ("UNITTEST/BTC", tick_interval), + ] def test_refresh(mocker, default_conf, ticker_history): refresh_mock = MagicMock() - mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', refresh_mock) + mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock) - exchange = get_patched_exchange(mocker, default_conf, id='binance') - tick_interval = default_conf['ticker_interval'] - pairs = [('XRP/BTC', tick_interval), - ('UNITTEST/BTC', tick_interval), - ] + exchange = get_patched_exchange(mocker, default_conf, id="binance") + tick_interval = default_conf["ticker_interval"] + pairs = [("XRP/BTC", tick_interval), ("UNITTEST/BTC", tick_interval)] - pairs_non_trad = [('ETH/USDT', tick_interval), - ('BTC/TUSD', "1h"), - ] + pairs_non_trad = [("ETH/USDT", tick_interval), ("BTC/TUSD", "1h")] dp = DataProvider(default_conf, exchange) dp.refresh(pairs) From 3221f883d34b112acf5991d31d353c1c1fa47e4d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 22 Jan 2019 06:58:51 +0100 Subject: [PATCH 42/49] Wrap line correctly --- freqtrade/exchange/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 62730e034..abbdae610 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -545,8 +545,9 @@ class Exchange(object): # Calculating ticker interval in second interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 - if not (self._pairs_last_refresh_time.get((pair, ticker_interval), 0) + interval_in_sec >= - arrow.utcnow().timestamp and (pair, ticker_interval) in self._klines): + if not ((self._pairs_last_refresh_time.get((pair, ticker_interval), 0) + + interval_in_sec) >= arrow.utcnow().timestamp + and (pair, ticker_interval) in self._klines): input_coroutines.append(self._async_get_candle_history(pair, ticker_interval)) else: logger.debug("Using cached ohlcv data for %s, %s ...", pair, ticker_interval) From c77607b9977cdb70014e0aa525a12304aa25b3fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 22 Jan 2019 07:38:15 +0100 Subject: [PATCH 43/49] Fix tests after rebase --- freqtrade/exchange/__init__.py | 2 +- freqtrade/tests/exchange/test_exchange.py | 46 ++++------------------- 2 files changed, 8 insertions(+), 40 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index abbdae610..e1aa0984a 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -594,7 +594,7 @@ class Exchange(object): data = sorted(data, key=lambda x: x[0]) except IndexError: logger.exception("Error loading %s. Result was %s.", pair, data) - return pair, [] + return pair, tick_interval, [] logger.debug("done fetching %s ...", pair) return pair, tick_interval, data diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 8fc3f1a32..a91b0ac56 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -805,7 +805,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: pairs = [('IOTA/ETH', '5m'), ('XRP/ETH', '5m')] # empty dicts assert not exchange._klines - exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m')]) + exchange.refresh_latest_ohlcv(pairs) assert log_has(f'Refreshing ohlcv data for {len(pairs)} pairs', caplog.record_tuples) assert exchange._klines @@ -892,42 +892,7 @@ async def test__async_get_candle_history_empty(default_conf, mocker, caplog): assert exchange._api_async.fetch_ohlcv.call_count == 1 -@pytest.mark.asyncio -async def test_async_get_candles_history(default_conf, mocker): - tick = [ - [ - 1511686200000, # unix timestamp ms - 1, # open - 2, # high - 3, # low - 4, # close - 5, # volume (in quote currency) - ] - ] - - async def mock_get_candle_hist(pair, tick_interval, since_ms=None): - return (pair, tick) - - exchange = get_patched_exchange(mocker, default_conf) - # Monkey-patch async function - exchange._api_async.fetch_ohlcv = get_mock_coro(tick) - - exchange._async_get_candle_history = Mock(wraps=mock_get_candle_hist) - - pairs = ['ETH/BTC', 'XRP/BTC'] - res = await exchange.async_get_candles_history(pairs, "5m") - assert type(res) is list - assert len(res) == 2 - assert type(res[0]) is tuple - assert res[0][0] == pairs[0] - assert res[0][1] == tick - assert res[1][0] == pairs[1] - assert res[1][1] == tick - assert exchange._async_get_candle_history.call_count == 2 - - -@pytest.mark.asyncio -async def test_async_get_candles_history_inv_result(default_conf, mocker, caplog): +def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog): async def mock_get_candle_hist(pair, *args, **kwargs): if pair == 'ETH/BTC': @@ -940,8 +905,11 @@ async def test_async_get_candles_history_inv_result(default_conf, mocker, caplog # Monkey-patch async function with empty result exchange._api_async.fetch_ohlcv = MagicMock(side_effect=mock_get_candle_hist) - pairs = ['ETH/BTC', 'XRP/BTC'] - res = await exchange.async_get_candles_history(pairs, "5m") + pairs = [("ETH/BTC", "5m"), ("XRP/BTC", "5m")] + res = exchange.refresh_latest_ohlcv(pairs) + assert exchange._klines + assert exchange._api_async.fetch_ohlcv.call_count == 2 + assert type(res) is list assert len(res) == 2 assert type(res[0]) is tuple From 89ddfe08f4ce241cc1f8a16be4f521dd80d74bd8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 22 Jan 2019 19:17:08 +0100 Subject: [PATCH 44/49] Add additional-pairs (sample) to defaultstrategy --- freqtrade/strategy/default_strategy.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 085a383db..36064f640 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -42,6 +42,19 @@ class DefaultStrategy(IStrategy): 'sell': 'gtc', } + def additional_pairs(self): + """ + Define additional pair/interval combinations to be cached from the exchange. + These pair/interval combinations are non-tradeable, unless they are part + of the whitelist as well. + For more information, please consult the documentation + :return: List of tuples in the format (pair, interval) + Sample: return [("ETH/USDT", "5m"), + ("BTC/USDT", "15m"), + ] + """ + return [] + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame From a06593e6e9495ca06c9dd19283575278e0b56c41 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 22 Jan 2019 19:17:21 +0100 Subject: [PATCH 45/49] Fix test --- freqtrade/tests/exchange/test_exchange.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index a91b0ac56..b384035b0 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -912,8 +912,9 @@ def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog): assert type(res) is list assert len(res) == 2 - assert type(res[0]) is tuple - assert type(res[1]) is TypeError + # Test that each is in list at least once as order is not guaranteed + assert type(res[0]) is tuple or type(res[1]) is tuple + assert type(res[0]) is TypeError or type(res[1]) is TypeError assert log_has("Error loading ETH/BTC. Result was [[]].", caplog.record_tuples) assert log_has("Async code raised an exception: TypeError", caplog.record_tuples) From 86a0863e30f3ff1e625c031aadf28dd91862f736 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 22 Jan 2019 19:25:58 +0100 Subject: [PATCH 46/49] Clarify logmessage Done fetching --- freqtrade/exchange/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index e1aa0984a..e4d83cf6d 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -595,7 +595,7 @@ class Exchange(object): except IndexError: logger.exception("Error loading %s. Result was %s.", pair, data) return pair, tick_interval, [] - logger.debug("done fetching %s ...", pair) + logger.debug("done fetching %s, %s ...", pair, tick_interval) return pair, tick_interval, data except ccxt.NotSupported as e: From bfd86093521478f788e208e4c0ce48c3ca54867f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Jan 2019 19:16:33 +0100 Subject: [PATCH 47/49] Fix comment --- freqtrade/data/dataprovider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index f69b1287d..375b8bf5b 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -63,7 +63,7 @@ class DataProvider(object): def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame: """ - get historic ohlcv data stored for backtesting + get stored historic ohlcv data :param pair: pair to get the data for :param tick_interval: ticker_interval to get pair for """ From ba07348b82ffc6c9be80fa2a62742bdc383c509d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Jan 2019 19:22:45 +0100 Subject: [PATCH 48/49] Rename additional_pairs to informative_pairs --- docs/bot-optimization.md | 4 ++-- freqtrade/freqtradebot.py | 2 +- freqtrade/strategy/default_strategy.py | 4 ++-- freqtrade/strategy/interface.py | 4 ++-- user_data/strategies/test_strategy.py | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/bot-optimization.md b/docs/bot-optimization.md index 5ac4be04c..8592f6cca 100644 --- a/docs/bot-optimization.md +++ b/docs/bot-optimization.md @@ -302,7 +302,7 @@ if self.dp: #### Get data for non-tradeable pairs -Data for additional pairs (reference pairs) can be beneficial for some strategies. +Data for additional, informative pairs (reference pairs) can be beneficial for some strategies. Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see above). These parts will **not** be traded unless they are also specified in the pair whitelist, or have been selected by Dynamic Whitelisting. @@ -311,7 +311,7 @@ The pairs need to be specified as tuples in the format `("pair", "interval")`, w Sample: ``` python -def additional_pairs(self): +def informative_pairs(self): return [("ETH/USDT", "5m"), ("BTC/TUSD", "15m"), ] diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 56ba404d6..656c700ac 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -175,7 +175,7 @@ class FreqtradeBot(object): for pair in self.active_pair_whitelist] # Refreshing candles self.dataprovider.refresh(pair_whitelist_tuple, - self.strategy.additional_pairs()) + self.strategy.informative_pairs()) # First process current opened trades for trade in trades: diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 36064f640..5c7d50a65 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -42,9 +42,9 @@ class DefaultStrategy(IStrategy): 'sell': 'gtc', } - def additional_pairs(self): + def informative_pairs(self): """ - Define additional pair/interval combinations to be cached from the exchange. + Define additional, informative pair/interval combinations to be cached from the exchange. These pair/interval combinations are non-tradeable, unless they are part of the whitelist as well. For more information, please consult the documentation diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index c7ec8dda6..733651df4 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -133,9 +133,9 @@ class IStrategy(ABC): :return: DataFrame with sell column """ - def additional_pairs(self) -> List[Tuple[str, str]]: + def informative_pairs(self) -> List[Tuple[str, str]]: """ - Define additional pair/interval combinations to be cached from the exchange. + Define additional, informative pair/interval combinations to be cached from the exchange. These pair/interval combinations are non-tradeable, unless they are part of the whitelist as well. For more information, please consult the documentation diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 314787f7a..e1f7d9c11 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -67,9 +67,9 @@ class TestStrategy(IStrategy): 'sell': 'gtc' } - def additional_pairs(self): + def informative_pairs(self): """ - Define additional pair/interval combinations to be cached from the exchange. + Define additional, informative pair/interval combinations to be cached from the exchange. These pair/interval combinations are non-tradeable, unless they are part of the whitelist as well. For more information, please consult the documentation From 3446dd179244097d8be4b25abeedcccb361364d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Jan 2019 20:05:49 +0100 Subject: [PATCH 49/49] Add test informative_pairs_added --- freqtrade/tests/test_freqtradebot.py | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index ef2815bed..9200c5fa6 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -807,6 +807,37 @@ def test_process_trade_no_whitelist_pair( assert result is True +def test_process_informative_pairs_added(default_conf, ticker, markets, mocker) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + + def _refresh_whitelist(list): + return ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'] + + refresh_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker, + get_markets=markets, + buy=MagicMock(side_effect=TemporaryError), + refresh_latest_ohlcv=refresh_mock, + ) + inf_pairs = MagicMock(return_value=[("BTC/ETH", '1m'), ("ETH/USDT", "1h")]) + mocker.patch('time.sleep', return_value=None) + + freqtrade = FreqtradeBot(default_conf) + freqtrade.pairlists._validate_whitelist = _refresh_whitelist + freqtrade.strategy.informative_pairs = inf_pairs + # patch_get_signal(freqtrade) + + freqtrade._process() + assert inf_pairs.call_count == 1 + assert refresh_mock.call_count == 1 + assert ("BTC/ETH", "1m") in refresh_mock.call_args[0][0] + assert ("ETH/USDT", "1h") in refresh_mock.call_args[0][0] + assert ("ETH/BTC", default_conf["ticker_interval"]) in refresh_mock.call_args[0][0] + + def test_balance_fully_ask_side(mocker, default_conf) -> None: default_conf['bid_strategy']['ask_last_balance'] = 0.0 freqtrade = get_patched_freqtradebot(mocker, default_conf)