From 0aa0b1d4feeb7ad3abe366d760852db879bbcd3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Dec 2018 07:15:21 +0100 Subject: [PATCH] Store tickers by pair / ticker_interval --- freqtrade/data/dataprovider.py | 18 ++++++++++++----- freqtrade/exchange/__init__.py | 32 ++++++++++++++++--------------- freqtrade/freqtradebot.py | 14 ++++++++++---- freqtrade/optimize/backtesting.py | 5 +++-- 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 628ca1dd4..9b9c73e55 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -6,7 +6,7 @@ Common Interface for bot and strategy to access data. """ import logging from pathlib import Path -from typing import List +from typing import List, Tuple from pandas import DataFrame @@ -23,11 +23,11 @@ class DataProvider(object): self._config = config self._exchange = exchange - def refresh(self, pairlist: List[str]) -> None: + def refresh(self, pairlist: List[Tuple[str, str]]) -> None: """ Refresh data, called with each cycle """ - self._exchange.refresh_latest_ohlcv(pairlist, self._config['ticker_interval']) + self._exchange.refresh_latest_ohlcv(pairlist) @property def available_pairs(self) -> List[str]: @@ -37,23 +37,31 @@ class DataProvider(object): """ return list(self._exchange._klines.keys()) - def ohlcv(self, pair: str, copy: bool = True) -> DataFrame: + def ohlcv(self, pair: str, tick_interval: str = None, copy: bool = True) -> DataFrame: """ get ohlcv data for the given pair as DataFrame Please check `available_pairs` to verify which pairs are currently cached. :param pair: pair to get the data for + :param tick_interval: ticker_interval to get pair for :param copy: copy dataframe before returning. Use false only for RO operations (where the dataframe is not modified) """ # TODO: Should not be stored in exchange but in this class if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - return self._exchange.klines(pair, copy) + if tick_interval: + pairtick = (pair, tick_interval) + else: + pairtick = (pair, self._config['ticker_interval']) + + return self._exchange.klines(pairtick, copy=copy) else: return DataFrame() def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame: """ get historic ohlcv data stored for backtesting + :param pair: pair to get the data for + :param tick_interval: ticker_interval to get pair for """ return load_pair_history(pair=pair, ticker_interval=ticker_interval, diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 9a426c805..21eab39d4 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -83,7 +83,7 @@ class Exchange(object): self._pairs_last_refresh_time: Dict[str, int] = {} # Holds candles - self._klines: Dict[str, DataFrame] = {} + self._klines: Dict[Tuple[str, str], DataFrame] = {} # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} @@ -158,9 +158,10 @@ class Exchange(object): """exchange ccxt id""" return self._api.id - def klines(self, pair: str, copy=True) -> DataFrame: - if pair in self._klines: - return self._klines[pair].copy() if copy else self._klines[pair] + def klines(self, pair_interval: Tuple[str, str], copy=True) -> DataFrame: + # create key tuple + if pair_interval in self._klines: + return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] else: return DataFrame() @@ -531,24 +532,24 @@ class Exchange(object): logger.info("downloaded %s with length %s.", pair, len(data)) return data - def refresh_latest_ohlcv(self, pair_list: List[str], - ticker_interval: str) -> List[Tuple[str, List]]: + def refresh_latest_ohlcv(self, pair_list: List[Tuple[str, str]]) -> List[Tuple[str, List]]: """ - Refresh in-memory ohlcv asyncronously and set `_klines` with the result + Refresh in-memory ohlcv asyncronously and set `_klines` with the result """ logger.debug("Refreshing ohlcv data for %d pairs", len(pair_list)) - # Calculating ticker interval in second - interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 input_coroutines = [] # Gather corotines to run - for pair in pair_list: + for pair, ticker_interval in pair_list: + # Calculating ticker interval in second + interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60 + if not (self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >= - arrow.utcnow().timestamp and pair in self._klines): + arrow.utcnow().timestamp and (pair, ticker_interval) in self._klines): input_coroutines.append(self._async_get_candle_history(pair, ticker_interval)) else: - logger.debug("Using cached ohlcv data for %s ...", pair) + logger.debug("Using cached ohlcv data for %s, %s ...", pair, ticker_interval) tickers = asyncio.get_event_loop().run_until_complete( asyncio.gather(*input_coroutines, return_exceptions=True)) @@ -559,13 +560,14 @@ class Exchange(object): logger.warning("Async code raised an exception: %s", res.__class__.__name__) continue pair = res[0] - tick_interval[1] + tick_interval = res[1] ticks = res[2] # keeping last candle time as last refreshed time of the pair if ticks: self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000 - # keeping parsed dataframe in cache - self._klines[pair] = parse_ticker_dataframe(ticks, tick_interval, fill_missing=True) + # keeping parsed dataframe in cache + self._klines[(pair, tick_interval)] = parse_ticker_dataframe( + ticks, tick_interval, fill_missing=True) return tickers @retrier_async diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6b27130bc..cc75fd89b 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -170,8 +170,11 @@ class FreqtradeBot(object): self.active_pair_whitelist.extend([trade.pair for trade in trades if trade.pair not in self.active_pair_whitelist]) + # Create pair-whitelist tuple with (pair, ticker_interval) + pair_whitelist = [(pair, self.config['ticker_interval']) + for pair in self.active_pair_whitelist] # Refreshing candles - self.dataprovider.refresh(self.active_pair_whitelist) + self.dataprovider.refresh(pair_whitelist) # First process current opened trades for trade in trades: @@ -321,7 +324,9 @@ class FreqtradeBot(object): # running get_signal on historical data fetched for _pair in whitelist: - (buy, sell) = self.strategy.get_signal(_pair, interval, self.dataprovider.ohlcv(_pair)) + (buy, sell) = self.strategy.get_signal( + _pair, interval, self.dataprovider.ohlcv(_pair, self.strategy.ticker_interval)) + if buy and not sell: stake_amount = self._get_trade_stake_amount(_pair) if not stake_amount: @@ -582,8 +587,9 @@ class FreqtradeBot(object): (buy, sell) = (False, False) experimental = self.config.get('experimental', {}) if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'): - (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval, - self.dataprovider.ohlcv(trade.pair)) + (buy, sell) = self.strategy.get_signal( + trade.pair, self.strategy.ticker_interval, + self.dataprovider.ohlcv(trade.pair, self.strategy.ticker_interval)) config_ask_strategy = self.config.get('ask_strategy', {}) if config_ask_strategy.get('use_order_book', False): diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 192f8cff0..a3e74704b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -368,8 +368,9 @@ class Backtesting(object): if self.config.get('live'): logger.info('Downloading data for all pairs in whitelist ...') - self.exchange.refresh_latest_ohlcv(pairs, self.ticker_interval) - data = self.exchange._klines + self.exchange.refresh_latest_ohlcv([(pair, self.ticker_interval) for pair in pairs]) + data = {key[0]: value for key, value in self.exchange._klines.items()} + else: logger.info('Using local backtesting data (using whitelist in given config) ...')