From 0ffb184ebad3631320e7f3279746e441d3c591a0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Aug 2019 20:45:24 +0200 Subject: [PATCH 1/9] Change some docstrings and formatting from history --- freqtrade/data/history.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index f600615df..363495796 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -57,10 +57,8 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: return tickerlist[start_index:stop_index] -def load_tickerdata_file( - datadir: Optional[Path], pair: str, - ticker_interval: str, - timerange: Optional[TimeRange] = None) -> Optional[list]: +def load_tickerdata_file(datadir: Optional[Path], pair: str, ticker_interval: str, + timerange: Optional[TimeRange] = None) -> Optional[list]: """ Load a pair from file, either .json.gz or .json :return: tickerlist or None if unsuccesful @@ -68,7 +66,7 @@ def load_tickerdata_file( filename = pair_data_filename(datadir, pair, ticker_interval) pairdata = misc.file_load_json(filename) if not pairdata: - return None + return [] if timerange: pairdata = trim_tickerlist(pairdata, timerange) @@ -182,6 +180,7 @@ def load_cached_data_for_updating(filename: Path, ticker_interval: str, Optional[int]]: """ Load cached data and choose what part of the data should be updated + Only used by download_pair_history(). """ since_ms = None From 91d1061c7304a598caa1be3a38c055530920956d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Aug 2019 20:48:42 +0200 Subject: [PATCH 2/9] Abstract tickerdata storing --- freqtrade/data/history.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 363495796..849b882b2 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -73,6 +73,15 @@ def load_tickerdata_file(datadir: Optional[Path], pair: str, ticker_interval: st return pairdata +def store_tickerdata_file(datadir: Optional[Path], pair: str, + ticker_interval: str, data: list, is_zip: bool = False): + """ + Stores tickerdata to file + """ + filename = pair_data_filename(datadir, pair, ticker_interval) + misc.file_dump_json(filename, data, is_zip=is_zip) + + def load_pair_history(pair: str, ticker_interval: str, datadir: Optional[Path], @@ -175,7 +184,7 @@ def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str) return filename -def load_cached_data_for_updating(filename: Path, ticker_interval: str, +def load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: """ @@ -194,12 +203,10 @@ def load_cached_data_for_updating(filename: Path, ticker_interval: str, since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 # read the cached file - if filename.is_file(): - with open(filename, "rt") as file: - data = misc.json_load(file) - # remove the last item, could be incomplete candle - if data: - data.pop() + data = load_tickerdata_file(datadir, pair, ticker_interval, TimeRange) + # remove the last item, could be incomplete candle + if data: + data.pop() else: data = [] @@ -238,14 +245,12 @@ def download_pair_history(datadir: Optional[Path], ) try: - filename = pair_data_filename(datadir, pair, ticker_interval) - logger.info( f'Download history data for pair: "{pair}", interval: {ticker_interval} ' f'and store in {datadir}.' ) - data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange) + data, since_ms = load_cached_data_for_updating(datadir, pair, ticker_interval, timerange) logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') @@ -260,7 +265,7 @@ def download_pair_history(datadir: Optional[Path], logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) - misc.file_dump_json(filename, data) + store_tickerdata_file(datadir, pair, ticker_interval, data=data) return True except Exception as e: From 9d3322df8c00d4d18f4b182ff0d494055c002013 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Aug 2019 20:49:13 +0200 Subject: [PATCH 3/9] Adapt history-tests to new load_cached_data header --- freqtrade/tests/data/test_history.py | 40 +++++++--------------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 00f4738f7..a06c5aa23 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -178,53 +178,41 @@ def test_load_cached_data_for_updating(mocker) -> None: # timeframe starts earlier than the cached data # should fully update data timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == [] assert start_ts == test_data[0][0] - 1000 # same with 'line' timeframe num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120 - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - TimeRange(None, 'line', 0, -num_lines)) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', TimeRange(None, 'line', 0, -num_lines)) assert data == [] assert start_ts < test_data[0][0] - 1 # timeframe starts in the center of the cached data # should return the chached data w/o the last item timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] # same with 'line' timeframe num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] # timeframe starts after the chached data # should return the chached data w/o the last item timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] # same with 'line' timeframe num_lines = 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] @@ -232,35 +220,27 @@ def test_load_cached_data_for_updating(mocker) -> None: # should return the chached data w/o the last item num_lines = 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] # no datafile exist # should return timestamp start time timerange = TimeRange('date', None, now_ts - 10000, 0) - data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'), - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange) assert data == [] assert start_ts == (now_ts - 10000) * 1000 # same with 'line' timeframe num_lines = 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'), - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange) assert data == [] assert start_ts == (now_ts - num_lines * 60) * 1000 # no datafile exist, no timeframe is set # should return an empty array and None - data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'), - '1m', - None) + data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', None) assert data == [] assert start_ts is None From b2a22f1afb4e10d635c33623acddd22a8dc4d9ae Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Aug 2019 21:39:53 +0200 Subject: [PATCH 4/9] Fix samll errors --- freqtrade/data/history.py | 4 ++-- freqtrade/tests/data/test_history.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 849b882b2..c6d731afa 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -184,7 +184,7 @@ def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str) return filename -def load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str, +def load_cached_data_for_updating(datadir: Optional[Path], pair: str, ticker_interval: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: """ @@ -203,7 +203,7 @@ def load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 # read the cached file - data = load_tickerdata_file(datadir, pair, ticker_interval, TimeRange) + data = load_tickerdata_file(datadir, pair, ticker_interval, timerange) # remove the last item, could be incomplete candle if data: data.pop() diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index a06c5aa23..164ebe01a 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -184,7 +184,8 @@ def test_load_cached_data_for_updating(mocker) -> None: # same with 'line' timeframe num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120 - data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', TimeRange(None, 'line', 0, -num_lines)) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', + TimeRange(None, 'line', 0, -num_lines)) assert data == [] assert start_ts < test_data[0][0] - 1 From f3e6bcb20c166bc7287caced4e3559604a39f0c4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Aug 2019 06:35:50 +0200 Subject: [PATCH 5/9] Avoid using negative indexes --- freqtrade/data/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index c6d731afa..5471767f6 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -43,7 +43,7 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: start_index += 1 if timerange.stoptype == 'line': - start_index = len(tickerlist) + timerange.stopts + start_index = max(len(tickerlist) + timerange.stopts, 0) if timerange.stoptype == 'index': stop_index = timerange.stopts elif timerange.stoptype == 'date': From a94a89086f7b37638a4eec19c93482db9d05a731 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Aug 2019 20:09:00 +0200 Subject: [PATCH 6/9] Don't forward timerange to load_ticker_file when loading cached data for updating. We always want to get all data, not just a fraction (we would end up overwriting the non-loaded part of the data). --- freqtrade/data/history.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 5471767f6..14749925f 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -203,7 +203,8 @@ def load_cached_data_for_updating(datadir: Optional[Path], pair: str, ticker_int since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 # read the cached file - data = load_tickerdata_file(datadir, pair, ticker_interval, timerange) + # Intentionally don't pass timerange in - since we need to load the full dataset. + data = load_tickerdata_file(datadir, pair, ticker_interval) # remove the last item, could be incomplete candle if data: data.pop() From 12677f2d42f8d3f789cd84b42f693bb142c6f9ad Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Aug 2019 20:13:19 +0200 Subject: [PATCH 7/9] Adjust docstring to match functioning of load_cached_data --- freqtrade/data/history.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 14749925f..af7d8cc7c 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -188,7 +188,9 @@ def load_cached_data_for_updating(datadir: Optional[Path], pair: str, ticker_int timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: """ - Load cached data and choose what part of the data should be updated + Load cached data to download more data. + If timerange is passed in, checks wether data from an before the stored data will be downloaded. + If that's the case than what's available should be completely overwritten. Only used by download_pair_history(). """ From 69eff890496df3cc374ae1f8eb260aa2d421db52 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Aug 2019 20:28:32 +0200 Subject: [PATCH 8/9] Improve comment in test_history to explain what is tested --- freqtrade/tests/data/test_history.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 164ebe01a..f238046d7 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -210,7 +210,8 @@ def test_load_cached_data_for_updating(mocker) -> None: assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] - # same with 'line' timeframe + # Try loading last 30 lines. + # Not supported by load_cached_data_for_updating, we always need to get the full data. num_lines = 30 timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) From 09286d49182fb98553dd2d137fc85b3d6b742a50 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 13:04:07 +0200 Subject: [PATCH 9/9] file_dump_json accepts Path - so we should feed it that --- freqtrade/misc.py | 8 ++++---- freqtrade/optimize/backtesting.py | 12 ++++++------ freqtrade/tests/optimize/test_backtesting.py | 5 +++-- freqtrade/tests/test_misc.py | 5 +++-- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 05946e008..d01d6a254 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -5,11 +5,11 @@ import gzip import logging import re from datetime import datetime +from pathlib import Path import numpy as np import rapidjson - logger = logging.getLogger(__name__) @@ -39,7 +39,7 @@ def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray: return dates.dt.to_pydatetime() -def file_dump_json(filename, data, is_zip=False) -> None: +def file_dump_json(filename: Path, data, is_zip=False) -> None: """ Dump JSON data into a file :param filename: file to create @@ -49,8 +49,8 @@ def file_dump_json(filename, data, is_zip=False) -> None: logger.info(f'dumping json to "{filename}"') if is_zip: - if not filename.endswith('.gz'): - filename = filename + '.gz' + if filename.suffix != '.gz': + filename = filename.with_suffix('.gz') with gzip.open(filename, 'w') as fp: rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) else: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 252175269..d321affeb 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -190,7 +190,7 @@ class Backtesting(object): return tabulate(tabular_data, headers=headers, # type: ignore floatfmt=floatfmt, tablefmt="pipe") - def _store_backtest_result(self, recordfilename: str, results: DataFrame, + def _store_backtest_result(self, recordfilename: Path, results: DataFrame, strategyname: Optional[str] = None) -> None: records = [(t.pair, t.profit_percent, t.open_time.timestamp(), @@ -201,10 +201,10 @@ class Backtesting(object): if records: if strategyname: # Inject strategyname to filename - recname = Path(recordfilename) - recordfilename = str(Path.joinpath( - recname.parent, f'{recname.stem}-{strategyname}').with_suffix(recname.suffix)) - logger.info('Dumping backtest results to %s', recordfilename) + recordfilename = Path.joinpath( + recordfilename.parent, + f'{recordfilename.stem}-{strategyname}').with_suffix(recordfilename.suffix) + logger.info(f'Dumping backtest results to {recordfilename}') file_dump_json(recordfilename, records) def _get_ticker_list(self, processed) -> Dict[str, DataFrame]: @@ -458,7 +458,7 @@ class Backtesting(object): for strategy, results in all_results.items(): if self.config.get('export', False): - self._store_backtest_result(self.config['exportfilename'], results, + self._store_backtest_result(Path(self.config['exportfilename']), results, strategy if len(self.strategylist) > 1 else None) print(f"Result for strategy {strategy}") diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 9ed7e7296..02e9a9c28 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -2,6 +2,7 @@ import math import random +from pathlib import Path from unittest.mock import MagicMock import numpy as np @@ -785,10 +786,10 @@ def test_backtest_record(default_conf, fee, mocker): # reset test to test with strategy name names = [] records = [] - backtesting._store_backtest_result("backtest-result.json", results, "DefStrat") + backtesting._store_backtest_result(Path("backtest-result.json"), results, "DefStrat") assert len(results) == 4 # Assert file_dump_json was only called once - assert names == ['backtest-result-DefStrat.json'] + assert names == [Path('backtest-result-DefStrat.json')] records = records[0] # Ensure records are of correct type assert len(records) == 4 diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 1a6b2a92d..c55083e64 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring,C0103 import datetime +from pathlib import Path from unittest.mock import MagicMock from freqtrade.data.converter import parse_ticker_dataframe @@ -34,12 +35,12 @@ def test_datesarray_to_datetimearray(ticker_history_list): def test_file_dump_json(mocker) -> None: file_open = mocker.patch('freqtrade.misc.open', MagicMock()) json_dump = mocker.patch('rapidjson.dump', MagicMock()) - file_dump_json('somefile', [1, 2, 3]) + file_dump_json(Path('somefile'), [1, 2, 3]) assert file_open.call_count == 1 assert json_dump.call_count == 1 file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock()) json_dump = mocker.patch('rapidjson.dump', MagicMock()) - file_dump_json('somefile', [1, 2, 3], True) + file_dump_json(Path('somefile'), [1, 2, 3], True) assert file_open.call_count == 1 assert json_dump.call_count == 1