From 02aacdd0c897cb0a6824b1d999bde2787622031f Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 29 Mar 2018 17:12:49 +0200 Subject: [PATCH 1/6] parse_ticker_dataframe: group dataframe by date --- freqtrade/analyze.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index ccdbb139e..4d5412d23 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -50,6 +50,14 @@ class Analyze(object): .rename(columns=columns) if 'BV' in frame: frame.drop('BV', 1, inplace=True) + # group by date to eliminate duplicate ticks + frame.groupby('date').agg({ + 'volume': 'max', + 'open': 'first', + 'close': 'last', + 'high': 'max', + 'low': 'min', + }) frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True) frame.sort_values('date', inplace=True) return frame From 4f2d3dbb418e7530446333f6c9a72410804ce040 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 29 Mar 2018 20:14:43 +0200 Subject: [PATCH 2/6] parse_ticker_dataframe: use as_index=False to keep date column --- freqtrade/analyze.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 4d5412d23..47edd289e 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -46,20 +46,20 @@ class Analyze(object): :return: DataFrame """ columns = {'C': 'close', 'V': 'volume', 'O': 'open', 'H': 'high', 'L': 'low', 'T': 'date'} - frame = DataFrame(ticker) \ - .rename(columns=columns) + frame = DataFrame(ticker).rename(columns=columns) if 'BV' in frame: - frame.drop('BV', 1, inplace=True) - # group by date to eliminate duplicate ticks - frame.groupby('date').agg({ - 'volume': 'max', - 'open': 'first', + frame.drop('BV', axis=1, inplace=True) + + frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True) + + # group by index and aggregate results to eliminate duplicate ticks + frame = frame.groupby(by='date', as_index=False, sort=True).agg({ 'close': 'last', 'high': 'max', 'low': 'min', + 'open': 'first', + 'volume': 'max', }) - frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True) - frame.sort_values('date', inplace=True) return frame def populate_indicators(self, dataframe: DataFrame) -> DataFrame: From 702402e1fefbfcdd849a95c854e766978c8cb0e1 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 29 Mar 2018 20:15:32 +0200 Subject: [PATCH 3/6] simplify download_backtesting_testdata --- freqtrade/optimize/__init__.py | 35 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index fff4b72d9..bfdfceab1 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -111,45 +111,38 @@ def download_pairs(datadir, pairs: List[str], ticker_interval: int) -> bool: # FIX: 20180110, suggest rename interval to tick_interval -def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> bool: +def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> None: """ Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters Based on @Rybolov work: https://github.com/rybolov/freqtrade-data - :param pairs: list of pairs to download - :return: bool """ path = make_testdata_path(datadir) logger.info( - 'Download the pair: "%s", Interval: %s min', - pair, - interval + 'Download the pair: "%s", Interval: %s min', pair, interval ) - filepair = pair.replace("-", "_") filename = os.path.join(path, '{pair}-{interval}.json'.format( - pair=filepair, + pair=pair.replace("-", "_"), interval=interval, )) if os.path.isfile(filename): with open(filename, "rt") as file: data = json.load(file) - logger.debug("Current Start: %s", data[1]['T']) - logger.debug("Current End: %s", data[-1:][0]['T']) else: data = [] - logger.debug("Current Start: None") - logger.debug("Current End: None") - new_data = get_ticker_history(pair=pair, tick_interval=int(interval)) - for row in new_data: - if row not in data: - data.append(row) - logger.debug("New Start: %s", data[1]['T']) - logger.debug("New End: %s", data[-1:][0]['T']) - data = sorted(data, key=lambda data: data['T']) + logger.debug('Current Start: %s', data[1]['T'] if data else None) + logger.debug('Current End: %s', data[-1:][0]['T'] if data else None) + # Extend data with new ticker history + data.extend([ + row for row in get_ticker_history(pair=pair, tick_interval=int(interval)) + if row not in data + ]) + + data = sorted(data, key=lambda _data: _data['T']) + logger.debug('New Start: %s', data[1]['T']) + logger.debug('New End: %s', data[-1:][0]['T']) misc.file_dump_json(filename, data) - - return True From fee8d0a2e1cfb40a95e66d730ac2e06def04c37c Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 29 Mar 2018 20:16:25 +0200 Subject: [PATCH 4/6] refactor get_timeframe --- freqtrade/optimize/backtesting.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9b71dbfd9..cf140637c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -4,11 +4,12 @@ This module contains the backtesting logic """ import logging +import operator from argparse import Namespace from typing import Dict, Tuple, Any, List, Optional import arrow -from pandas import DataFrame, Series +from pandas import DataFrame from tabulate import tabulate import freqtrade.optimize as optimize @@ -60,11 +61,12 @@ class Backtesting(object): :param data: dictionary with preprocessed backtesting data :return: tuple containing min_date, max_date """ - all_dates = Series([]) - for pair_data in data.values(): - all_dates = all_dates.append(pair_data['date']) - all_dates.sort_values(inplace=True) - return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1]) + timeframe = [ + (arrow.get(min(frame.date)), arrow.get(max(frame.date))) + for frame in data.values() + ] + return min(timeframe, key=operator.itemgetter(0))[0], \ + max(timeframe, key=operator.itemgetter(1))[1] def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: """ From 3775fdf9c798b450c9517eb82ed8b1248f0ba4a5 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 29 Mar 2018 20:16:46 +0200 Subject: [PATCH 5/6] change column order assertions --- freqtrade/tests/test_analyze.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_analyze.py b/freqtrade/tests/test_analyze.py index a4f1ba549..878d97950 100644 --- a/freqtrade/tests/test_analyze.py +++ b/freqtrade/tests/test_analyze.py @@ -50,7 +50,7 @@ def test_dataframe_correct_length(result): def test_dataframe_correct_columns(result): assert result.columns.tolist() == \ - ['close', 'high', 'low', 'open', 'date', 'volume'] + ['date', 'close', 'high', 'low', 'open', 'volume'] def test_populates_buy_trend(result): @@ -170,7 +170,7 @@ def test_get_signal_handles_exceptions(mocker): def test_parse_ticker_dataframe(ticker_history, ticker_history_without_bv): - columns = ['close', 'high', 'low', 'open', 'date', 'volume'] + columns = ['date', 'close', 'high', 'low', 'open', 'volume'] # Test file with BV data dataframe = Analyze.parse_ticker_dataframe(ticker_history) From 24aa6a1679d63c0409b0de922afb3e8afa13fbb2 Mon Sep 17 00:00:00 2001 From: gcarq Date: Thu, 29 Mar 2018 20:17:11 +0200 Subject: [PATCH 6/6] adapt test_download_backtesting_testdata --- freqtrade/tests/optimize/test_optimize.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/optimize/test_optimize.py b/freqtrade/tests/optimize/test_optimize.py index e26d30534..a15bff5a1 100644 --- a/freqtrade/tests/optimize/test_optimize.py +++ b/freqtrade/tests/optimize/test_optimize.py @@ -182,10 +182,11 @@ def test_download_backtesting_testdata(ticker_history, mocker) -> None: def test_download_backtesting_testdata2(mocker) -> None: tick = [{'T': 'bar'}, {'T': 'foo'}] - mocker.patch('freqtrade.misc.file_dump_json', return_value=None) + json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=tick) - assert download_backtesting_testdata(None, pair="BTC-UNITEST", interval=1) - assert download_backtesting_testdata(None, pair="BTC-UNITEST", interval=3) + download_backtesting_testdata(None, pair="BTC-UNITEST", interval=1) + download_backtesting_testdata(None, pair="BTC-UNITEST", interval=3) + assert json_dump_mock.call_count == 2 def test_load_tickerdata_file() -> None: