diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index ccdbb139e..47edd289e 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -46,12 +46,20 @@ class Analyze(object): :return: DataFrame """ columns = {'C': 'close', 'V': 'volume', 'O': 'open', 'H': 'high', 'L': 'low', 'T': 'date'} - frame = DataFrame(ticker) \ - .rename(columns=columns) + frame = DataFrame(ticker).rename(columns=columns) if 'BV' in frame: - frame.drop('BV', 1, inplace=True) + frame.drop('BV', axis=1, inplace=True) + frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True) - frame.sort_values('date', inplace=True) + + # group by index and aggregate results to eliminate duplicate ticks + frame = frame.groupby(by='date', as_index=False, sort=True).agg({ + 'close': 'last', + 'high': 'max', + 'low': 'min', + 'open': 'first', + 'volume': 'max', + }) return frame def populate_indicators(self, dataframe: DataFrame) -> DataFrame: diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index fff4b72d9..bfdfceab1 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -111,45 +111,38 @@ def download_pairs(datadir, pairs: List[str], ticker_interval: int) -> bool: # FIX: 20180110, suggest rename interval to tick_interval -def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> bool: +def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> None: """ Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters Based on @Rybolov work: https://github.com/rybolov/freqtrade-data - :param pairs: list of pairs to download - :return: bool """ path = make_testdata_path(datadir) logger.info( - 'Download the pair: "%s", Interval: %s min', - pair, - interval + 'Download the pair: "%s", Interval: %s min', pair, interval ) - filepair = pair.replace("-", "_") filename = os.path.join(path, '{pair}-{interval}.json'.format( - pair=filepair, + pair=pair.replace("-", "_"), interval=interval, )) if os.path.isfile(filename): with open(filename, "rt") as file: data = json.load(file) - logger.debug("Current Start: %s", data[1]['T']) - logger.debug("Current End: %s", data[-1:][0]['T']) else: data = [] - logger.debug("Current Start: None") - logger.debug("Current End: None") - new_data = get_ticker_history(pair=pair, tick_interval=int(interval)) - for row in new_data: - if row not in data: - data.append(row) - logger.debug("New Start: %s", data[1]['T']) - logger.debug("New End: %s", data[-1:][0]['T']) - data = sorted(data, key=lambda data: data['T']) + logger.debug('Current Start: %s', data[1]['T'] if data else None) + logger.debug('Current End: %s', data[-1:][0]['T'] if data else None) + # Extend data with new ticker history + data.extend([ + row for row in get_ticker_history(pair=pair, tick_interval=int(interval)) + if row not in data + ]) + + data = sorted(data, key=lambda _data: _data['T']) + logger.debug('New Start: %s', data[1]['T']) + logger.debug('New End: %s', data[-1:][0]['T']) misc.file_dump_json(filename, data) - - return True diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9b71dbfd9..cf140637c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -4,11 +4,12 @@ This module contains the backtesting logic """ import logging +import operator from argparse import Namespace from typing import Dict, Tuple, Any, List, Optional import arrow -from pandas import DataFrame, Series +from pandas import DataFrame from tabulate import tabulate import freqtrade.optimize as optimize @@ -60,11 +61,12 @@ class Backtesting(object): :param data: dictionary with preprocessed backtesting data :return: tuple containing min_date, max_date """ - all_dates = Series([]) - for pair_data in data.values(): - all_dates = all_dates.append(pair_data['date']) - all_dates.sort_values(inplace=True) - return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1]) + timeframe = [ + (arrow.get(min(frame.date)), arrow.get(max(frame.date))) + for frame in data.values() + ] + return min(timeframe, key=operator.itemgetter(0))[0], \ + max(timeframe, key=operator.itemgetter(1))[1] def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: """ diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index e26d30534..a15bff5a1 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -182,10 +182,11 @@ def test_download_backtesting_testdata(ticker_history, mocker) -> None: def test_download_backtesting_testdata2(mocker) -> None: tick = [{'T': 'bar'}, {'T': 'foo'}] - mocker.patch('freqtrade.misc.file_dump_json', return_value=None) + json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=tick) - assert download_backtesting_testdata(None, pair="BTC-UNITEST", interval=1) - assert download_backtesting_testdata(None, pair="BTC-UNITEST", interval=3) + download_backtesting_testdata(None, pair="BTC-UNITEST", interval=1) + download_backtesting_testdata(None, pair="BTC-UNITEST", interval=3) + assert json_dump_mock.call_count == 2 def test_load_tickerdata_file() -> None: diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index a4f1ba549..878d97950 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -50,7 +50,7 @@ def test_dataframe_correct_length(result): def test_dataframe_correct_columns(result): assert result.columns.tolist() == \ - ['close', 'high', 'low', 'open', 'date', 'volume'] + ['date', 'close', 'high', 'low', 'open', 'volume'] def test_populates_buy_trend(result): @@ -170,7 +170,7 @@ def test_get_signal_handles_exceptions(mocker): def test_parse_ticker_dataframe(ticker_history, ticker_history_without_bv): - columns = ['close', 'high', 'low', 'open', 'date', 'volume'] + columns = ['date', 'close', 'high', 'low', 'open', 'volume'] # Test file with BV data dataframe = Analyze.parse_ticker_dataframe(ticker_history)