adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Store tickers by pair / ticker_interval

Browse Source
This commit is contained in:
Matthias 2018-12-30 07:15:21 +01:00
parent 5f61da30ed
commit 0aa0b1d4fe
4 changed files with 43 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
freqtrade/data/dataprovider.py
View File

@ -6,7 +6,7 @@ Common Interface for bot and strategy to access data.
"""
import logging
from pathlib import Path
from typing import List
from typing import List, Tuple
from pandas import DataFrame
@ -23,11 +23,11 @@ class DataProvider(object):
self._config = config
self._exchange = exchange
def refresh(self, pairlist: List[str]) -> None:
def refresh(self, pairlist: List[Tuple[str, str]]) -> None:
"""
Refresh data, called with each cycle
"""
self._exchange.refresh_latest_ohlcv(pairlist, self._config['ticker_interval'])
self._exchange.refresh_latest_ohlcv(pairlist)
@property
def available_pairs(self) -> List[str]:
@ -37,23 +37,31 @@ class DataProvider(object):
"""
return list(self._exchange._klines.keys())
def ohlcv(self, pair: str, copy: bool = True) -> DataFrame:
def ohlcv(self, pair: str, tick_interval: str = None, copy: bool = True) -> DataFrame:
"""
get ohlcv data for the given pair as DataFrame
Please check `available_pairs` to verify which pairs are currently cached.
:param pair: pair to get the data for
:param tick_interval: ticker_interval to get pair for
:param copy: copy dataframe before returning.
Use false only for RO operations (where the dataframe is not modified)
"""
# TODO: Should not be stored in exchange but in this class
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
return self._exchange.klines(pair, copy)
if tick_interval:
pairtick = (pair, tick_interval)
else:
pairtick = (pair, self._config['ticker_interval'])
return self._exchange.klines(pairtick, copy=copy)
else:
return DataFrame()
def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame:
"""
get historic ohlcv data stored for backtesting
:param pair: pair to get the data for
:param tick_interval: ticker_interval to get pair for
"""
return load_pair_history(pair=pair,
ticker_interval=ticker_interval,

32
freqtrade/exchange/__init__.py
View File

@ -83,7 +83,7 @@ class Exchange(object):
self._pairs_last_refresh_time: Dict[str, int] = {}
# Holds candles
self._klines: Dict[str, DataFrame] = {}
self._klines: Dict[Tuple[str, str], DataFrame] = {}
# Holds all open sell orders for dry_run
self._dry_run_open_orders: Dict[str, Any] = {}
@ -158,9 +158,10 @@ class Exchange(object):
"""exchange ccxt id"""
return self._api.id
def klines(self, pair: str, copy=True) -> DataFrame:
if pair in self._klines:
return self._klines[pair].copy() if copy else self._klines[pair]
def klines(self, pair_interval: Tuple[str, str], copy=True) -> DataFrame:
# create key tuple
if pair_interval in self._klines:
return self._klines[pair_interval].copy() if copy else self._klines[pair_interval]
else:
return DataFrame()
@ -531,24 +532,24 @@ class Exchange(object):
logger.info("downloaded %s with length %s.", pair, len(data))
return data
def refresh_latest_ohlcv(self, pair_list: List[str],
ticker_interval: str) -> List[Tuple[str, List]]:
def refresh_latest_ohlcv(self, pair_list: List[Tuple[str, str]]) -> List[Tuple[str, List]]:
"""
Refresh in-memory ohlcv asyncronously and set `_klines` with the result
Refresh in-memory ohlcv asyncronously and set `_klines` with the result
"""
logger.debug("Refreshing ohlcv data for %d pairs", len(pair_list))
# Calculating ticker interval in second
interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60
input_coroutines = []
# Gather corotines to run
for pair in pair_list:
for pair, ticker_interval in pair_list:
# Calculating ticker interval in second
interval_in_sec = constants.TICKER_INTERVAL_MINUTES[ticker_interval] * 60
if not (self._pairs_last_refresh_time.get(pair, 0) + interval_in_sec >=
arrow.utcnow().timestamp and pair in self._klines):
arrow.utcnow().timestamp and (pair, ticker_interval) in self._klines):
input_coroutines.append(self._async_get_candle_history(pair, ticker_interval))
else:
logger.debug("Using cached ohlcv data for %s ...", pair)
logger.debug("Using cached ohlcv data for %s, %s ...", pair, ticker_interval)
tickers = asyncio.get_event_loop().run_until_complete(
asyncio.gather(*input_coroutines, return_exceptions=True))
@ -559,13 +560,14 @@ class Exchange(object):
logger.warning("Async code raised an exception: %s", res.__class__.__name__)
continue
pair = res[0]
tick_interval[1]
tick_interval = res[1]
ticks = res[2]
# keeping last candle time as last refreshed time of the pair
if ticks:
self._pairs_last_refresh_time[pair] = ticks[-1][0] // 1000
# keeping parsed dataframe in cache
self._klines[pair] = parse_ticker_dataframe(ticks, tick_interval, fill_missing=True)
# keeping parsed dataframe in cache
self._klines[(pair, tick_interval)] = parse_ticker_dataframe(
ticks, tick_interval, fill_missing=True)
return tickers
@retrier_async

14
freqtrade/freqtradebot.py
View File

@ -170,8 +170,11 @@ class FreqtradeBot(object):
self.active_pair_whitelist.extend([trade.pair for trade in trades
if trade.pair not in self.active_pair_whitelist])
# Create pair-whitelist tuple with (pair, ticker_interval)
pair_whitelist = [(pair, self.config['ticker_interval'])
for pair in self.active_pair_whitelist]
# Refreshing candles
self.dataprovider.refresh(self.active_pair_whitelist)
self.dataprovider.refresh(pair_whitelist)
# First process current opened trades
for trade in trades:
@ -321,7 +324,9 @@ class FreqtradeBot(object):
# running get_signal on historical data fetched
for _pair in whitelist:
(buy, sell) = self.strategy.get_signal(_pair, interval, self.dataprovider.ohlcv(_pair))
(buy, sell) = self.strategy.get_signal(
_pair, interval, self.dataprovider.ohlcv(_pair, self.strategy.ticker_interval))
if buy and not sell:
stake_amount = self._get_trade_stake_amount(_pair)
if not stake_amount:
@ -582,8 +587,9 @@ class FreqtradeBot(object):
(buy, sell) = (False, False)
experimental = self.config.get('experimental', {})
if experimental.get('use_sell_signal') or experimental.get('ignore_roi_if_buy_signal'):
(buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.ticker_interval,
self.dataprovider.ohlcv(trade.pair))
(buy, sell) = self.strategy.get_signal(
trade.pair, self.strategy.ticker_interval,
self.dataprovider.ohlcv(trade.pair, self.strategy.ticker_interval))
config_ask_strategy = self.config.get('ask_strategy', {})
if config_ask_strategy.get('use_order_book', False):

5
freqtrade/optimize/backtesting.py
View File

@ -368,8 +368,9 @@ class Backtesting(object):
if self.config.get('live'):
logger.info('Downloading data for all pairs in whitelist ...')
self.exchange.refresh_latest_ohlcv(pairs, self.ticker_interval)
data = self.exchange._klines
self.exchange.refresh_latest_ohlcv([(pair, self.ticker_interval) for pair in pairs])
data = {key[0]: value for key, value in self.exchange._klines.items()}
else:
logger.info('Using local backtesting data (using whitelist in given config) ...')