From 89ff614e1d7e5deb04347849ba5b2f3060b180de Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Jun 2019 13:46:19 +0200 Subject: [PATCH] Add pair as parameter, and warn when fillup was necessary --- freqtrade/data/converter.py | 8 ++++---- freqtrade/data/history.py | 2 +- freqtrade/exchange/exchange.py | 2 +- freqtrade/optimize/hyperopt.py | 7 ++----- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index d1b8277ea..43c91a843 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -10,7 +10,7 @@ from pandas import DataFrame, to_datetime logger = logging.getLogger(__name__) -def parse_ticker_dataframe(ticker: list, ticker_interval: str, *, +def parse_ticker_dataframe(ticker: list, ticker_interval: str, pair: str, *, fill_missing: bool = True, drop_incomplete: bool = True) -> DataFrame: """ @@ -51,12 +51,12 @@ def parse_ticker_dataframe(ticker: list, ticker_interval: str, *, logger.debug('Dropping last candle') if fill_missing: - return ohlcv_fill_up_missing_data(frame, ticker_interval) + return ohlcv_fill_up_missing_data(frame, ticker_interval, pair) else: return frame -def ohlcv_fill_up_missing_data(dataframe: DataFrame, ticker_interval: str) -> DataFrame: +def ohlcv_fill_up_missing_data(dataframe: DataFrame, ticker_interval: str, pair: str) -> DataFrame: """ Fills up missing data with 0 volume rows, using the previous close as price for "open", "high" "low" and "close", volume is set to 0 @@ -87,7 +87,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, ticker_interval: str) -> Da len_before = len(dataframe) len_after = len(df) if len_before != len_after: - logger.info(f"Missing data fillup: before: {len_before} - after: {len_after}") + logger.info(f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}") return df diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 83ea84284..655034b33 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -116,7 +116,7 @@ def load_pair_history(pair: str, logger.warning('Missing data at end for pair %s, data ends at %s', pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - return parse_ticker_dataframe(pairdata, ticker_interval, + return parse_ticker_dataframe(pairdata, ticker_interval, pair=pair, fill_missing=fill_up_missing, drop_incomplete=drop_incomplete) else: diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ea6996efb..db264d1bc 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -581,7 +581,7 @@ class Exchange(object): self._pairs_last_refresh_time[(pair, ticker_interval)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache self._klines[(pair, ticker_interval)] = parse_ticker_dataframe( - ticks, ticker_interval, fill_missing=True, + ticks, ticker_interval, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) return tickers diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 28b9ce789..7fd9bf5d9 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -19,8 +19,7 @@ from skopt import Optimizer from skopt.space import Dimension from freqtrade.arguments import Arguments -from freqtrade.data.history import load_data, get_timeframe, validate_backtest_data -from freqtrade.exchange import timeframe_to_minutes +from freqtrade.data.history import load_data, get_timeframe from freqtrade.optimize.backtesting import Backtesting from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver @@ -281,9 +280,7 @@ class Hyperopt(Backtesting): return min_date, max_date = get_timeframe(data) - # Validate dataframe for missing values (mainly at start and end, as fillup is called) - validate_backtest_data(data, min_date, max_date, - timeframe_to_minutes(self.ticker_interval)) + logger.info( 'Hyperopting with data from %s up to %s (%s days)..', min_date.isoformat(),