From 0ffb184ebad3631320e7f3279746e441d3c591a0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Aug 2019 20:45:24 +0200 Subject: [PATCH 01/42] Change some docstrings and formatting from history --- freqtrade/data/history.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index f600615df..363495796 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -57,10 +57,8 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: return tickerlist[start_index:stop_index] -def load_tickerdata_file( - datadir: Optional[Path], pair: str, - ticker_interval: str, - timerange: Optional[TimeRange] = None) -> Optional[list]: +def load_tickerdata_file(datadir: Optional[Path], pair: str, ticker_interval: str, + timerange: Optional[TimeRange] = None) -> Optional[list]: """ Load a pair from file, either .json.gz or .json :return: tickerlist or None if unsuccesful @@ -68,7 +66,7 @@ def load_tickerdata_file( filename = pair_data_filename(datadir, pair, ticker_interval) pairdata = misc.file_load_json(filename) if not pairdata: - return None + return [] if timerange: pairdata = trim_tickerlist(pairdata, timerange) @@ -182,6 +180,7 @@ def load_cached_data_for_updating(filename: Path, ticker_interval: str, Optional[int]]: """ Load cached data and choose what part of the data should be updated + Only used by download_pair_history(). """ since_ms = None From 91d1061c7304a598caa1be3a38c055530920956d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Aug 2019 20:48:42 +0200 Subject: [PATCH 02/42] Abstract tickerdata storing --- freqtrade/data/history.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 363495796..849b882b2 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -73,6 +73,15 @@ def load_tickerdata_file(datadir: Optional[Path], pair: str, ticker_interval: st return pairdata +def store_tickerdata_file(datadir: Optional[Path], pair: str, + ticker_interval: str, data: list, is_zip: bool = False): + """ + Stores tickerdata to file + """ + filename = pair_data_filename(datadir, pair, ticker_interval) + misc.file_dump_json(filename, data, is_zip=is_zip) + + def load_pair_history(pair: str, ticker_interval: str, datadir: Optional[Path], @@ -175,7 +184,7 @@ def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str) return filename -def load_cached_data_for_updating(filename: Path, ticker_interval: str, +def load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: """ @@ -194,12 +203,10 @@ def load_cached_data_for_updating(filename: Path, ticker_interval: str, since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 # read the cached file - if filename.is_file(): - with open(filename, "rt") as file: - data = misc.json_load(file) - # remove the last item, could be incomplete candle - if data: - data.pop() + data = load_tickerdata_file(datadir, pair, ticker_interval, TimeRange) + # remove the last item, could be incomplete candle + if data: + data.pop() else: data = [] @@ -238,14 +245,12 @@ def download_pair_history(datadir: Optional[Path], ) try: - filename = pair_data_filename(datadir, pair, ticker_interval) - logger.info( f'Download history data for pair: "{pair}", interval: {ticker_interval} ' f'and store in {datadir}.' ) - data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange) + data, since_ms = load_cached_data_for_updating(datadir, pair, ticker_interval, timerange) logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') @@ -260,7 +265,7 @@ def download_pair_history(datadir: Optional[Path], logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) - misc.file_dump_json(filename, data) + store_tickerdata_file(datadir, pair, ticker_interval, data=data) return True except Exception as e: From 9d3322df8c00d4d18f4b182ff0d494055c002013 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Aug 2019 20:49:13 +0200 Subject: [PATCH 03/42] Adapt history-tests to new load_cached_data header --- freqtrade/tests/data/test_history.py | 40 +++++++--------------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 00f4738f7..a06c5aa23 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -178,53 +178,41 @@ def test_load_cached_data_for_updating(mocker) -> None: # timeframe starts earlier than the cached data # should fully update data timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == [] assert start_ts == test_data[0][0] - 1000 # same with 'line' timeframe num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120 - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - TimeRange(None, 'line', 0, -num_lines)) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', TimeRange(None, 'line', 0, -num_lines)) assert data == [] assert start_ts < test_data[0][0] - 1 # timeframe starts in the center of the cached data # should return the chached data w/o the last item timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] # same with 'line' timeframe num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] # timeframe starts after the chached data # should return the chached data w/o the last item timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] # same with 'line' timeframe num_lines = 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] @@ -232,35 +220,27 @@ def test_load_cached_data_for_updating(mocker) -> None: # should return the chached data w/o the last item num_lines = 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] # no datafile exist # should return timestamp start time timerange = TimeRange('date', None, now_ts - 10000, 0) - data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'), - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange) assert data == [] assert start_ts == (now_ts - 10000) * 1000 # same with 'line' timeframe num_lines = 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'), - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange) assert data == [] assert start_ts == (now_ts - num_lines * 60) * 1000 # no datafile exist, no timeframe is set # should return an empty array and None - data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'), - '1m', - None) + data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', None) assert data == [] assert start_ts is None From b2a22f1afb4e10d635c33623acddd22a8dc4d9ae Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Aug 2019 21:39:53 +0200 Subject: [PATCH 04/42] Fix samll errors --- freqtrade/data/history.py | 4 ++-- freqtrade/tests/data/test_history.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 849b882b2..c6d731afa 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -184,7 +184,7 @@ def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str) return filename -def load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str, +def load_cached_data_for_updating(datadir: Optional[Path], pair: str, ticker_interval: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: """ @@ -203,7 +203,7 @@ def load_cached_data_for_updating(datadir: Path, pair: str, ticker_interval: str since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 # read the cached file - data = load_tickerdata_file(datadir, pair, ticker_interval, TimeRange) + data = load_tickerdata_file(datadir, pair, ticker_interval, timerange) # remove the last item, could be incomplete candle if data: data.pop() diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index a06c5aa23..164ebe01a 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -184,7 +184,8 @@ def test_load_cached_data_for_updating(mocker) -> None: # same with 'line' timeframe num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120 - data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', TimeRange(None, 'line', 0, -num_lines)) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', + TimeRange(None, 'line', 0, -num_lines)) assert data == [] assert start_ts < test_data[0][0] - 1 From f3e6bcb20c166bc7287caced4e3559604a39f0c4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Aug 2019 06:35:50 +0200 Subject: [PATCH 05/42] Avoid using negative indexes --- freqtrade/data/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index c6d731afa..5471767f6 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -43,7 +43,7 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: start_index += 1 if timerange.stoptype == 'line': - start_index = len(tickerlist) + timerange.stopts + start_index = max(len(tickerlist) + timerange.stopts, 0) if timerange.stoptype == 'index': stop_index = timerange.stopts elif timerange.stoptype == 'date': From a94a89086f7b37638a4eec19c93482db9d05a731 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Aug 2019 20:09:00 +0200 Subject: [PATCH 06/42] Don't forward timerange to load_ticker_file when loading cached data for updating. We always want to get all data, not just a fraction (we would end up overwriting the non-loaded part of the data). --- freqtrade/data/history.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 5471767f6..14749925f 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -203,7 +203,8 @@ def load_cached_data_for_updating(datadir: Optional[Path], pair: str, ticker_int since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 # read the cached file - data = load_tickerdata_file(datadir, pair, ticker_interval, timerange) + # Intentionally don't pass timerange in - since we need to load the full dataset. + data = load_tickerdata_file(datadir, pair, ticker_interval) # remove the last item, could be incomplete candle if data: data.pop() From 12677f2d42f8d3f789cd84b42f693bb142c6f9ad Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Aug 2019 20:13:19 +0200 Subject: [PATCH 07/42] Adjust docstring to match functioning of load_cached_data --- freqtrade/data/history.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 14749925f..af7d8cc7c 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -188,7 +188,9 @@ def load_cached_data_for_updating(datadir: Optional[Path], pair: str, ticker_int timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: """ - Load cached data and choose what part of the data should be updated + Load cached data to download more data. + If timerange is passed in, checks wether data from an before the stored data will be downloaded. + If that's the case than what's available should be completely overwritten. Only used by download_pair_history(). """ From 69eff890496df3cc374ae1f8eb260aa2d421db52 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Aug 2019 20:28:32 +0200 Subject: [PATCH 08/42] Improve comment in test_history to explain what is tested --- freqtrade/tests/data/test_history.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 164ebe01a..f238046d7 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -210,7 +210,8 @@ def test_load_cached_data_for_updating(mocker) -> None: assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] - # same with 'line' timeframe + # Try loading last 30 lines. + # Not supported by load_cached_data_for_updating, we always need to get the full data. num_lines = 30 timerange = TimeRange(None, 'line', 0, -num_lines) data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) From 09286d49182fb98553dd2d137fc85b3d6b742a50 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 13:04:07 +0200 Subject: [PATCH 09/42] file_dump_json accepts Path - so we should feed it that --- freqtrade/misc.py | 8 ++++---- freqtrade/optimize/backtesting.py | 12 ++++++------ freqtrade/tests/optimize/test_backtesting.py | 5 +++-- freqtrade/tests/test_misc.py | 5 +++-- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 05946e008..d01d6a254 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -5,11 +5,11 @@ import gzip import logging import re from datetime import datetime +from pathlib import Path import numpy as np import rapidjson - logger = logging.getLogger(__name__) @@ -39,7 +39,7 @@ def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray: return dates.dt.to_pydatetime() -def file_dump_json(filename, data, is_zip=False) -> None: +def file_dump_json(filename: Path, data, is_zip=False) -> None: """ Dump JSON data into a file :param filename: file to create @@ -49,8 +49,8 @@ def file_dump_json(filename, data, is_zip=False) -> None: logger.info(f'dumping json to "{filename}"') if is_zip: - if not filename.endswith('.gz'): - filename = filename + '.gz' + if filename.suffix != '.gz': + filename = filename.with_suffix('.gz') with gzip.open(filename, 'w') as fp: rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) else: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 252175269..d321affeb 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -190,7 +190,7 @@ class Backtesting(object): return tabulate(tabular_data, headers=headers, # type: ignore floatfmt=floatfmt, tablefmt="pipe") - def _store_backtest_result(self, recordfilename: str, results: DataFrame, + def _store_backtest_result(self, recordfilename: Path, results: DataFrame, strategyname: Optional[str] = None) -> None: records = [(t.pair, t.profit_percent, t.open_time.timestamp(), @@ -201,10 +201,10 @@ class Backtesting(object): if records: if strategyname: # Inject strategyname to filename - recname = Path(recordfilename) - recordfilename = str(Path.joinpath( - recname.parent, f'{recname.stem}-{strategyname}').with_suffix(recname.suffix)) - logger.info('Dumping backtest results to %s', recordfilename) + recordfilename = Path.joinpath( + recordfilename.parent, + f'{recordfilename.stem}-{strategyname}').with_suffix(recordfilename.suffix) + logger.info(f'Dumping backtest results to {recordfilename}') file_dump_json(recordfilename, records) def _get_ticker_list(self, processed) -> Dict[str, DataFrame]: @@ -458,7 +458,7 @@ class Backtesting(object): for strategy, results in all_results.items(): if self.config.get('export', False): - self._store_backtest_result(self.config['exportfilename'], results, + self._store_backtest_result(Path(self.config['exportfilename']), results, strategy if len(self.strategylist) > 1 else None) print(f"Result for strategy {strategy}") diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 9ed7e7296..02e9a9c28 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -2,6 +2,7 @@ import math import random +from pathlib import Path from unittest.mock import MagicMock import numpy as np @@ -785,10 +786,10 @@ def test_backtest_record(default_conf, fee, mocker): # reset test to test with strategy name names = [] records = [] - backtesting._store_backtest_result("backtest-result.json", results, "DefStrat") + backtesting._store_backtest_result(Path("backtest-result.json"), results, "DefStrat") assert len(results) == 4 # Assert file_dump_json was only called once - assert names == ['backtest-result-DefStrat.json'] + assert names == [Path('backtest-result-DefStrat.json')] records = records[0] # Ensure records are of correct type assert len(records) == 4 diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 1a6b2a92d..c55083e64 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring,C0103 import datetime +from pathlib import Path from unittest.mock import MagicMock from freqtrade.data.converter import parse_ticker_dataframe @@ -34,12 +35,12 @@ def test_datesarray_to_datetimearray(ticker_history_list): def test_file_dump_json(mocker) -> None: file_open = mocker.patch('freqtrade.misc.open', MagicMock()) json_dump = mocker.patch('rapidjson.dump', MagicMock()) - file_dump_json('somefile', [1, 2, 3]) + file_dump_json(Path('somefile'), [1, 2, 3]) assert file_open.call_count == 1 assert json_dump.call_count == 1 file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock()) json_dump = mocker.patch('rapidjson.dump', MagicMock()) - file_dump_json('somefile', [1, 2, 3], True) + file_dump_json(Path('somefile'), [1, 2, 3], True) assert file_open.call_count == 1 assert json_dump.call_count == 1 From 91886120a7d17071a4663cd39c36d378b7d563c1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 14:37:10 +0200 Subject: [PATCH 10/42] use nargs for --pairs argument --- freqtrade/configuration/cli_options.py | 6 ++++-- freqtrade/constants.py | 1 - freqtrade/plot/plotting.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index 84686d1e6..d39013737 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -254,7 +254,8 @@ AVAILABLE_CLI_OPTIONS = { # Script options "pairs": Arg( '-p', '--pairs', - help='Show profits for only these pairs. Pairs are comma-separated.', + help='Show profits for only these pairs. Pairs are space-separated.', + nargs='+', ), # Download data "pairs_file": Arg( @@ -276,9 +277,10 @@ AVAILABLE_CLI_OPTIONS = { "timeframes": Arg( '-t', '--timeframes', help=f'Specify which tickers to download. Space-separated list. ' - f'Default: `{constants.DEFAULT_DOWNLOAD_TICKER_INTERVALS}`.', + f'Default: `1m 5m`.', choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h', '1d', '3d', '1w'], + default=['1m', '5m'], nargs='+', ), "erase": Arg( diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 9b73adcfe..fbf44dec8 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -23,7 +23,6 @@ ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList'] DRY_RUN_WALLET = 999.9 -DEFAULT_DOWNLOAD_TICKER_INTERVALS = '1m 5m' TICKER_INTERVALS = [ '1m', '3m', '5m', '15m', '30m', diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 947b3003c..e6da581a4 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -37,7 +37,7 @@ def init_plotscript(config): strategy = StrategyResolver(config).strategy if "pairs" in config: - pairs = config["pairs"].split(',') + pairs = config["pairs"] else: pairs = config["exchange"]["pair_whitelist"] From 05deb9e09bdccd0c19904ed2687cc6bd8f2bf29f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 14:42:44 +0200 Subject: [PATCH 11/42] Migrate download-script logic to utils.py --- freqtrade/configuration/arguments.py | 19 ++++++-- freqtrade/tests/test_arguments.py | 18 ++++---- freqtrade/utils.py | 66 ++++++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 16 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 926d02f8f..8fa16318a 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -30,7 +30,7 @@ ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] ARGS_LIST_EXCHANGES = ["print_one_column"] -ARGS_DOWNLOADER = ARGS_COMMON + ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] +ARGS_DOWNLOADER = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", @@ -40,6 +40,8 @@ ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + ARGS_PLOT_PROFIT = (ARGS_COMMON + ARGS_STRATEGY + ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source"]) +NO_CONF_REQURIED = ["start_download_data"] + class Arguments(object): """ @@ -75,7 +77,10 @@ class Arguments(object): # Workaround issue in argparse with action='append' and default value # (see https://bugs.python.org/issue16399) - if not self._no_default_config and parsed_arg.config is None: + # Allow no-config for certain commands (like downloading / plotting) + if (not self._no_default_config and parsed_arg.config is None + and not (hasattr(parsed_arg, 'func') + and parsed_arg.func.__name__ in NO_CONF_REQURIED)): parsed_arg.config = [constants.DEFAULT_CONFIG] return parsed_arg @@ -93,7 +98,7 @@ class Arguments(object): :return: None """ from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge - from freqtrade.utils import start_list_exchanges + from freqtrade.utils import start_download_data, start_list_exchanges subparsers = self.parser.add_subparsers(dest='subparser') @@ -119,3 +124,11 @@ class Arguments(object): ) list_exchanges_cmd.set_defaults(func=start_list_exchanges) self._build_args(optionlist=ARGS_LIST_EXCHANGES, parser=list_exchanges_cmd) + + # Add download-data subcommand + download_data_cmd = subparsers.add_parser( + 'download-data', + help='Download backtesting data.' + ) + download_data_cmd.set_defaults(func=start_download_data) + self._build_args(optionlist=ARGS_DOWNLOADER, parser=download_data_cmd) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 2cb7ff6d7..31ab9dea8 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -50,10 +50,10 @@ def test_parse_args_verbose() -> None: def test_common_scripts_options() -> None: - arguments = Arguments(['-p', 'ETH/BTC'], '') - arguments._build_args(ARGS_DOWNLOADER) - args = arguments._parse_args() - assert args.pairs == 'ETH/BTC' + args = Arguments(['download-data', '-p', 'ETH/BTC', 'XRP/BTC'], '').get_parsed_arg() + + assert args.pairs == ['ETH/BTC', 'XRP/BTC'] + assert hasattr(args, "func") def test_parse_args_version() -> None: @@ -135,14 +135,14 @@ def test_parse_args_hyperopt_custom() -> None: def test_download_data_options() -> None: args = [ - '--pairs-file', 'file_with_pairs', '--datadir', 'datadir/directory', + 'download-data', + '--pairs-file', 'file_with_pairs', '--days', '30', '--exchange', 'binance' ] - arguments = Arguments(args, '') - arguments._build_args(ARGS_DOWNLOADER) - args = arguments._parse_args() + args = Arguments(args, '').get_parsed_arg() + assert args.pairs_file == 'file_with_pairs' assert args.datadir == 'datadir/directory' assert args.days == 30 @@ -162,7 +162,7 @@ def test_plot_dataframe_options() -> None: assert pargs.indicators1 == "sma10,sma100" assert pargs.indicators2 == "macd,fastd,fastk" assert pargs.plot_limit == 30 - assert pargs.pairs == "UNITTEST/BTC" + assert pargs.pairs == ["UNITTEST/BTC"] def test_check_int_positive() -> None: diff --git a/freqtrade/utils.py b/freqtrade/utils.py index d550ef43c..d2770ba1a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -1,11 +1,16 @@ import logging +import sys from argparse import Namespace +from pathlib import Path from typing import Any, Dict -from freqtrade.configuration import Configuration -from freqtrade.exchange import available_exchanges -from freqtrade.state import RunMode +import arrow +from freqtrade.configuration import Configuration, TimeRange +from freqtrade.data.history import download_pair_history +from freqtrade.exchange import available_exchanges +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -17,7 +22,7 @@ def setup_utils_configuration(args: Namespace, method: RunMode) -> Dict[str, Any :return: Configuration """ configuration = Configuration(args, method) - config = configuration.load_config() + config = configuration.get_config() config['exchange']['dry_run'] = True # Ensure we do not use Exchange credentials @@ -39,3 +44,56 @@ def start_list_exchanges(args: Namespace) -> None: else: print(f"Exchanges supported by ccxt and available for Freqtrade: " f"{', '.join(available_exchanges())}") + + +def start_download_data(args: Namespace) -> None: + """ + Download data based + """ + config = setup_utils_configuration(args, RunMode.OTHER) + + timerange = TimeRange() + if 'days' in config: + time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + + dl_path = Path(config['datadir']) + logger.info(f'About to download pairs: {config["pairs"]}, ' + f'intervals: {config["timeframes"]} to {dl_path}') + + pairs_not_available = [] + + try: + # Init exchange + exchange = ExchangeResolver(config['exchange']['name'], config).exchange + + for pair in config["pairs"]: + if pair not in exchange._api.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + for ticker_interval in config["timeframes"]: + pair_print = pair.replace('/', '_') + filename = f'{pair_print}-{ticker_interval}.json' + dl_file = dl_path.joinpath(filename) + if args.erase and dl_file.exists(): + logger.info( + f'Deleting existing data for pair {pair}, interval {ticker_interval}.') + dl_file.unlink() + + logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') + download_pair_history(datadir=dl_path, exchange=exchange, + pair=pair, ticker_interval=str(ticker_interval), + timerange=timerange) + + except KeyboardInterrupt: + sys.exit("SIGINT received, aborting ...") + + finally: + if pairs_not_available: + logger.info( + f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {config['exchange']['name']}.") + + # configuration.resolve_pairs_list() + print(config) From 8655e521d7c64812ffda7f69e95e4959a6fcf6f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 14:53:46 +0200 Subject: [PATCH 12/42] Adapt some tests --- freqtrade/tests/test_arguments.py | 2 +- freqtrade/tests/test_main.py | 4 +++- freqtrade/tests/test_plotting.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 31ab9dea8..601f41e63 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -4,7 +4,7 @@ import argparse import pytest from freqtrade.configuration import Arguments -from freqtrade.configuration.arguments import ARGS_DOWNLOADER, ARGS_PLOT_DATAFRAME +from freqtrade.configuration.arguments import ARGS_PLOT_DATAFRAME from freqtrade.configuration.cli_options import check_int_positive diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index d8ec532b0..409025a3c 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -1,7 +1,7 @@ # pragma pylint: disable=missing-docstring from copy import deepcopy -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock import pytest @@ -21,6 +21,7 @@ def test_parse_args_backtesting(mocker) -> None: further argument parsing is done in test_arguments.py """ backtesting_mock = mocker.patch('freqtrade.optimize.start_backtesting', MagicMock()) + backtesting_mock.__name__ = PropertyMock("start_backtesting") # it's sys.exit(0) at the end of backtesting with pytest.raises(SystemExit): main(['backtesting']) @@ -36,6 +37,7 @@ def test_parse_args_backtesting(mocker) -> None: def test_main_start_hyperopt(mocker) -> None: hyperopt_mock = mocker.patch('freqtrade.optimize.start_hyperopt', MagicMock()) + hyperopt_mock.__name__ = PropertyMock("start_hyperopt") # it's sys.exit(0) at the end of hyperopt with pytest.raises(SystemExit): main(['hyperopt']) diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index cd72160f8..94d40ab84 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -50,7 +50,7 @@ def test_init_plotscript(default_conf, mocker): assert "pairs" in ret assert "strategy" in ret - default_conf['pairs'] = "POWR/BTC,XLM/BTC" + default_conf['pairs'] = ["POWR/BTC", "XLM/BTC"] ret = init_plotscript(default_conf) assert "tickers" in ret assert "POWR/BTC" in ret["tickers"] From 3c15e3ebddd07d14ba085f4d226d5bea2d7a97a9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 14:56:38 +0200 Subject: [PATCH 13/42] Default load minimal config --- freqtrade/configuration/configuration.py | 13 +++++++++++++ freqtrade/constants.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index c95246fc0..e24ace34b 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -52,6 +52,9 @@ class Configuration(object): # Keep this method as staticmethod, so it can be used from interactive environments config: Dict[str, Any] = {} + if not files: + return constants.MINIMAL_CONFIG + # We expect here a list of config filenames for path in files: logger.info(f'Using config: {path} ...') @@ -276,6 +279,16 @@ class Configuration(object): self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') + self._args_to_config(config, argname='timeframes', + logstring='timeframes --timeframes: {}') + + self._args_to_config(config, argname='days', + logstring='Detected --days: {}') + + if "exchange" in self.args and self.args.exchange: + config['exchange']['name'] = self.args.exchange + logger.info(f"Using exchange {config['exchange']['name']}") + def _process_runmode(self, config: Dict[str, Any]) -> None: if not self.runmode: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index fbf44dec8..b73a723eb 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -38,6 +38,20 @@ SUPPORTED_FIAT = [ "BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT" ] +MINIMAL_CONFIG = { + 'stake_currency': '', + 'dry_run': True, + 'exchange': { + 'name': '', + 'key': '', + 'secret': '', + 'pair_whitelist': [], + 'ccxt_async_config': { + 'enableRateLimit': True, + } + } +} + # Required json-schema for user specified config CONF_SCHEMA = { 'type': 'object', From 4e308a1a3e4aef76c809e53f1d60953399fb8655 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 14:56:57 +0200 Subject: [PATCH 14/42] Resolve pairlist in configuration --- freqtrade/configuration/configuration.py | 37 ++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index e24ace34b..ed12b6501 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -90,6 +90,11 @@ class Configuration(object): self._process_runmode(config) + # Check if the exchange set by the user is supported + check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) + + self._resolve_pairs_list(config) + return config def _process_logging_options(self, config: Dict[str, Any]) -> None: @@ -150,9 +155,6 @@ class Configuration(object): if 'sd_notify' in self.args and self.args.sd_notify: config['internals'].update({'sd_notify': True}) - # Check if the exchange set by the user is supported - check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) - def _process_datadir_options(self, config: Dict[str, Any]) -> None: """ Extract information for sys.argv and load datadir configuration: @@ -348,3 +350,32 @@ class Configuration(object): logger.info(logstring.format(config[argname])) if deprecated_msg: warnings.warn(f"DEPRECATED: {deprecated_msg}", DeprecationWarning) + + def _resolve_pairs_list(self, config: Dict[str, Any]) -> None: + """ + Helper for download script. + Takes first found: + * -p (pairs argument) + * --pairs-file + * whitelist from config + """ + + if "pairs" in self.args and self.args.pairs: + return + + if "pairs_file" in self.args and self.args.pairs_file: + pairs_file = self.args.pairs_file + logger.info(f'Reading pairs file "{pairs_file}".') + # Download pairs from the pairs file if no config is specified + # or if pairs file is specified explicitely + if not pairs_file.exists(): + OperationalException(f'No pairs file found with path "{pairs_file}".') + + # with pairs_file.open() as file: + # pairs = list(set(json.load(file))) + + # pairs.sort() + + if "config" in self.args: + logger.info("Using pairlist from configuration.") + config['pairs'] = config.get('exchange', {}).get('pair_whitelist') From 219d0b7fb016f51f90f5778941d6aba72f907910 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 15:27:33 +0200 Subject: [PATCH 15/42] Adjust documentation to removed download-script --- docs/backtesting.md | 67 +++++++++++++----------- freqtrade/configuration/configuration.py | 3 ++ freqtrade/data/history.py | 2 +- freqtrade/tests/data/test_history.py | 4 +- freqtrade/utils.py | 4 +- 5 files changed, 43 insertions(+), 37 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 7e9f7ff53..f666c5b49 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -3,9 +3,43 @@ This page explains how to validate your strategy performance by using Backtesting. +## Getting data for backtesting / hyperopt + +To download backtesting data (candles / OHLCV), we recommend using the `freqtrade download-data` command. + +If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes. +Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. + +Alternatively, a `pairs.json` file can be used. + +If you are using Binance for example: + +- create a directory `user_data/data/binance` and copy `pairs.json` in that directory. +- update the `pairs.json` to contain the currency pairs you are interested in. + +```bash +mkdir -p user_data/data/binance +cp freqtrade/tests/testdata/pairs.json user_data/data/binance +``` + +Then run: + +```bash +freqtrade download-data --exchange binance +``` + +This will download ticker data for all the currency pairs you defined in `pairs.json`. + +- To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. +- To change the exchange used to download the tickers, please use a different configuration file (you'll probably need to adjust ratelimits etc.) +- To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. +- To download ticker data for only 10 days, use `--days 10` (defaults to 30 days). +- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. +- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. + ## Test your strategy with Backtesting -Now you have good Buy and Sell strategies, you want to test it against +Now you have good Buy and Sell strategies and some historic data, you want to test it against real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). @@ -109,37 +143,6 @@ The full timerange specification: - Use tickframes between POSIX timestamps 1527595200 1527618600: `--timerange=1527595200-1527618600` -#### Downloading new set of ticker data - -To download new set of backtesting ticker data, you can use a download script. - -If you are using Binance for example: - -- create a directory `user_data/data/binance` and copy `pairs.json` in that directory. -- update the `pairs.json` to contain the currency pairs you are interested in. - -```bash -mkdir -p user_data/data/binance -cp freqtrade/tests/testdata/pairs.json user_data/data/binance -``` - -Then run: - -```bash -python scripts/download_backtest_data.py --exchange binance -``` - -This will download ticker data for all the currency pairs you defined in `pairs.json`. - -- To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. -- To change the exchange used to download the tickers, use `--exchange`. Default is `bittrex`. -- To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. -- To download ticker data for only 10 days, use `--days 10`. -- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. -- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with other options. - -For help about backtesting usage, please refer to [Backtesting commands](#backtesting-commands). - ## Understand the backtesting result The most important in the backtesting is to understand the result. diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index ed12b6501..f7c393b60 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -281,6 +281,9 @@ class Configuration(object): self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') + self._args_to_config(config, argname='erase', + logstring='Erase detected. Deleting existing data.') + self._args_to_config(config, argname='timeframes', logstring='timeframes --timeframes: {}') diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 899c6d0c8..c7b3a28b0 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -122,7 +122,7 @@ def load_pair_history(pair: str, else: logger.warning( f'No history data for pair: "{pair}", interval: {ticker_interval}. ' - 'Use --refresh-pairs-cached option or download_backtest_data.py ' + 'Use --refresh-pairs-cached option or `freqtrade download-data` ' 'script to download the data' ) return None diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 4ba65e470..ea56b4bec 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -74,7 +74,7 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None: assert ld is None assert log_has( 'No history data for pair: "UNITTEST/BTC", interval: 7m. ' - 'Use --refresh-pairs-cached option or download_backtest_data.py ' + 'Use --refresh-pairs-cached option or `freqtrade download-data` ' 'script to download the data', caplog ) @@ -109,7 +109,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau assert os.path.isfile(file) is False assert log_has( 'No history data for pair: "MEME/BTC", interval: 1m. ' - 'Use --refresh-pairs-cached option or download_backtest_data.py ' + 'Use --refresh-pairs-cached option or `freqtrade download-data` ' 'script to download the data', caplog ) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index d2770ba1a..7ccbae81a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -48,7 +48,7 @@ def start_list_exchanges(args: Namespace) -> None: def start_download_data(args: Namespace) -> None: """ - Download data based + Download data (former download_backtest_data.py script) """ config = setup_utils_configuration(args, RunMode.OTHER) @@ -76,7 +76,7 @@ def start_download_data(args: Namespace) -> None: pair_print = pair.replace('/', '_') filename = f'{pair_print}-{ticker_interval}.json' dl_file = dl_path.joinpath(filename) - if args.erase and dl_file.exists(): + if config.get("erase") and dl_file.exists(): logger.info( f'Deleting existing data for pair {pair}, interval {ticker_interval}.') dl_file.unlink() From 89257832d721cb06c7e2f80f860f8006fc856fc8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 15:27:59 +0200 Subject: [PATCH 16/42] Don't use internal _API methods --- freqtrade/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 7ccbae81a..002d89738 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -68,7 +68,7 @@ def start_download_data(args: Namespace) -> None: exchange = ExchangeResolver(config['exchange']['name'], config).exchange for pair in config["pairs"]: - if pair not in exchange._api.markets: + if pair not in exchange.markets: pairs_not_available.append(pair) logger.info(f"Skipping pair {pair}...") continue From b2c215029d4937223b88b39a7d37c2373896be0d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 15:28:11 +0200 Subject: [PATCH 17/42] Add tests for download_data entrypoint --- freqtrade/tests/test_utils.py | 51 ++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_utils.py b/freqtrade/tests/test_utils.py index a12b709d7..8d0e76cde 100644 --- a/freqtrade/tests/test_utils.py +++ b/freqtrade/tests/test_utils.py @@ -1,8 +1,11 @@ -from freqtrade.utils import setup_utils_configuration, start_list_exchanges -from freqtrade.tests.conftest import get_args -from freqtrade.state import RunMode - import re +from pathlib import Path +from unittest.mock import MagicMock, PropertyMock + +from freqtrade.state import RunMode +from freqtrade.tests.conftest import get_args, log_has, patch_exchange +from freqtrade.utils import (setup_utils_configuration, start_download_data, + start_list_exchanges) def test_setup_utils_configuration(): @@ -40,3 +43,43 @@ def test_list_exchanges(capsys): assert not re.match(r"Exchanges supported by ccxt and available.*", captured.out) assert re.search(r"^binance$", captured.out, re.MULTILINE) assert re.search(r"^bittrex$", captured.out, re.MULTILINE) + + +def test_download_data(mocker, markets, caplog): + dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch.object(Path, "unlink", MagicMock()) + + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--erase" + ] + start_download_data(get_args(args)) + + assert dl_mock.call_count == 4 + assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog) + assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) + + +def test_download_data_no_markets(mocker, caplog): + dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC" + ] + start_download_data(get_args(args)) + + assert dl_mock.call_count == 0 + assert log_has("Skipping pair ETH/BTC...", caplog) + assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) From 132f28ad44be07194ee68cda69a488c75d1b7b8b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 15:52:59 +0200 Subject: [PATCH 18/42] Add tests to correctly load / override pair-lists --- freqtrade/configuration/configuration.py | 21 ++++-- freqtrade/tests/test_configuration.py | 88 ++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 7 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index f7c393b60..cb698544d 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -4,6 +4,7 @@ This module contains the configuration class import logging import warnings from argparse import Namespace +from pathlib import Path from typing import Any, Callable, Dict, List, Optional from freqtrade import OperationalException, constants @@ -12,7 +13,7 @@ from freqtrade.configuration.create_datadir import create_datadir from freqtrade.configuration.json_schema import validate_config_schema from freqtrade.configuration.load_config import load_config_file from freqtrade.loggers import setup_logging -from freqtrade.misc import deep_merge_dicts +from freqtrade.misc import deep_merge_dicts, json_load from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -363,22 +364,28 @@ class Configuration(object): * whitelist from config """ - if "pairs" in self.args and self.args.pairs: + if "pairs" in config: return if "pairs_file" in self.args and self.args.pairs_file: - pairs_file = self.args.pairs_file + pairs_file = Path(self.args.pairs_file) logger.info(f'Reading pairs file "{pairs_file}".') # Download pairs from the pairs file if no config is specified # or if pairs file is specified explicitely if not pairs_file.exists(): - OperationalException(f'No pairs file found with path "{pairs_file}".') + raise OperationalException(f'No pairs file found with path "{pairs_file}".') - # with pairs_file.open() as file: - # pairs = list(set(json.load(file))) + config['pairs'] = json_load(pairs_file) - # pairs.sort() + config['pairs'].sort() + return if "config" in self.args: logger.info("Using pairlist from configuration.") config['pairs'] = config.get('exchange', {}).get('pair_whitelist') + else: + # Fall back to /dl_path/pairs.json + pairs_file = Path(config['datadir']) / "pairs.json" + if pairs_file.exists(): + config['pairs'] = json_load(pairs_file) + diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 8cbd02ece..c351b9b72 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -704,3 +704,91 @@ def test_load_config_default_subkeys(all_conf, keys) -> None: validate_config_schema(all_conf) assert subkey in all_conf[key] assert all_conf[key][subkey] == keys[2] + + +def test_pairlist_resolving(): + arglist = [ + 'download-data', + '--pairs', 'ETH/BTC', 'XRP/BTC', + '--exchange', 'binance' + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] + assert config['exchange']['name'] == 'binance' + + +def test_pairlist_resolving_with_config(mocker, default_conf): + patched_configuration_load_config_file(mocker, default_conf) + arglist = [ + '--config', 'config.json', + 'download-data', + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert config['pairs'] == default_conf['exchange']['pair_whitelist'] + assert config['exchange']['name'] == default_conf['exchange']['name'] + + # Override pairs + arglist = [ + '--config', 'config.json', + 'download-data', + '--pairs', 'ETH/BTC', 'XRP/BTC', + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] + assert config['exchange']['name'] == default_conf['exchange']['name'] + + +def test_pairlist_resolving_with_config_pl(mocker, default_conf): + patched_configuration_load_config_file(mocker, default_conf) + load_mock = mocker.patch("freqtrade.configuration.configuration.json_load", + MagicMock(return_value=['XRP/BTC', 'ETH/BTC'])) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + + arglist = [ + '--config', 'config.json', + 'download-data', + '--pairs-file', 'pairs.json', + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert load_mock.call_count == 1 + assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] + assert config['exchange']['name'] == default_conf['exchange']['name'] + + +def test_pairlist_resolving_with_config_pl_not_exists(mocker, default_conf): + patched_configuration_load_config_file(mocker, default_conf) + mocker.patch("freqtrade.configuration.configuration.json_load", + MagicMock(return_value=['XRP/BTC', 'ETH/BTC'])) + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + + arglist = [ + '--config', 'config.json', + 'download-data', + '--pairs-file', 'pairs.json', + ] + + args = Arguments(arglist, '').get_parsed_arg() + + with pytest.raises(OperationalException, match=r"No pairs file found with path.*"): + configuration = Configuration(args) + configuration.get_config() From c9207bcc0070bd574bc301f984955bbf3cdda0cf Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 16:01:30 +0200 Subject: [PATCH 19/42] Remove blank line at end --- freqtrade/configuration/configuration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index cb698544d..c51153f4b 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -388,4 +388,3 @@ class Configuration(object): pairs_file = Path(config['datadir']) / "pairs.json" if pairs_file.exists(): config['pairs'] = json_load(pairs_file) - From 29c56f4447ad53cb1d9dc157227be4ea09b4ab29 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Aug 2019 06:47:53 +0200 Subject: [PATCH 20/42] Replace download_backtest_data script with warning message --- scripts/download_backtest_data.py | 145 ++---------------------------- 1 file changed, 5 insertions(+), 140 deletions(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index f77ad7422..496f83c7d 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -1,144 +1,9 @@ -#!/usr/bin/env python3 -""" -This script generates json files with pairs history data -""" -import arrow -import json import sys -from pathlib import Path -from typing import Any, Dict, List -from freqtrade.configuration import Arguments, TimeRange -from freqtrade.configuration import Configuration -from freqtrade.configuration.arguments import ARGS_DOWNLOADER -from freqtrade.configuration.check_exchange import check_exchange -from freqtrade.configuration.load_config import load_config_file -from freqtrade.data.history import download_pair_history -from freqtrade.exchange import Exchange -from freqtrade.misc import deep_merge_dicts -import logging +print("This script has been integrated into freqtrade " + "and it's functionality is available by calling `freqtrade download-data`.") +print("Please check the documentation on https://www.freqtrade.io/en/latest/backtesting/ " + "for details.") -logger = logging.getLogger('download_backtest_data') - -DEFAULT_DL_PATH = 'user_data/data' - -# Do not read the default config if config is not specified -# in the command line options explicitely -arguments = Arguments(sys.argv[1:], 'Download backtest data', - no_default_config=True) -arguments._build_args(optionlist=ARGS_DOWNLOADER) -args = arguments._parse_args() - -# Use bittrex as default exchange -exchange_name = args.exchange or 'bittrex' - -pairs: List = [] - -configuration = Configuration(args) -config: Dict[str, Any] = {} - -if args.config: - # Now expecting a list of config filenames here, not a string - for path in args.config: - logger.info(f"Using config: {path}...") - # Merge config options, overwriting old values - config = deep_merge_dicts(load_config_file(path), config) - - config['stake_currency'] = '' - # Ensure we do not use Exchange credentials - config['exchange']['dry_run'] = True - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - - pairs = config['exchange']['pair_whitelist'] - - if config.get('ticker_interval'): - timeframes = args.timeframes or [config.get('ticker_interval')] - else: - timeframes = args.timeframes or ['1m', '5m'] - -else: - config = { - 'stake_currency': '', - 'dry_run': True, - 'exchange': { - 'name': exchange_name, - 'key': '', - 'secret': '', - 'pair_whitelist': [], - 'ccxt_async_config': { - 'enableRateLimit': True, - 'rateLimit': 200 - } - } - } - timeframes = args.timeframes or ['1m', '5m'] - -configuration._process_logging_options(config) - -if args.config and args.exchange: - logger.warning("The --exchange option is ignored, " - "using exchange settings from the configuration file.") - -# Check if the exchange set by the user is supported -check_exchange(config) - -configuration._process_datadir_options(config) - -dl_path = Path(config['datadir']) - -pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json') - -if not pairs or args.pairs_file: - logger.info(f'Reading pairs file "{pairs_file}".') - # Download pairs from the pairs file if no config is specified - # or if pairs file is specified explicitely - if not pairs_file.exists(): - sys.exit(f'No pairs file found with path "{pairs_file}".') - - with pairs_file.open() as file: - pairs = list(set(json.load(file))) - - pairs.sort() - -timerange = TimeRange() -if args.days: - time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d") - timerange = TimeRange.parse_timerange(f'{time_since}-') - -logger.info(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}') - -pairs_not_available = [] - -try: - # Init exchange - exchange = Exchange(config) - - for pair in pairs: - if pair not in exchange._api.markets: - pairs_not_available.append(pair) - logger.info(f"Skipping pair {pair}...") - continue - for ticker_interval in timeframes: - pair_print = pair.replace('/', '_') - filename = f'{pair_print}-{ticker_interval}.json' - dl_file = dl_path.joinpath(filename) - if args.erase and dl_file.exists(): - logger.info( - f'Deleting existing data for pair {pair}, interval {ticker_interval}.') - dl_file.unlink() - - logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') - download_pair_history(datadir=dl_path, exchange=exchange, - pair=pair, ticker_interval=str(ticker_interval), - timerange=timerange) - -except KeyboardInterrupt: - sys.exit("SIGINT received, aborting ...") - -finally: - if pairs_not_available: - logger.info( - f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {config['exchange']['name']}.") +sys.exit(1) From f7d5280f47ced954a57536cde7dc1cc6561f7203 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Aug 2019 06:48:34 +0200 Subject: [PATCH 21/42] Replace ARGS_DOWNLOADER with ARGS_DOWNLOAD_DATA --- freqtrade/configuration/arguments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 8fa16318a..c45e3d7ba 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -30,7 +30,7 @@ ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] ARGS_LIST_EXCHANGES = ["print_one_column"] -ARGS_DOWNLOADER = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] +ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", @@ -131,4 +131,4 @@ class Arguments(object): help='Download backtesting data.' ) download_data_cmd.set_defaults(func=start_download_data) - self._build_args(optionlist=ARGS_DOWNLOADER, parser=download_data_cmd) + self._build_args(optionlist=ARGS_DOWNLOAD_DATA, parser=download_data_cmd) From a53e9e3a98cf440d1f14a9d55d907a4139e46b97 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Aug 2019 06:58:38 +0200 Subject: [PATCH 22/42] improve tests for download_module --- freqtrade/tests/test_utils.py | 50 +++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_utils.py b/freqtrade/tests/test_utils.py index 8d0e76cde..003b3b286 100644 --- a/freqtrade/tests/test_utils.py +++ b/freqtrade/tests/test_utils.py @@ -2,6 +2,8 @@ import re from pathlib import Path from unittest.mock import MagicMock, PropertyMock +import pytest + from freqtrade.state import RunMode from freqtrade.tests.conftest import get_args, log_has, patch_exchange from freqtrade.utils import (setup_utils_configuration, start_download_data, @@ -58,15 +60,41 @@ def test_download_data(mocker, markets, caplog): "download-data", "--exchange", "binance", "--pairs", "ETH/BTC", "XRP/BTC", - "--erase" + "--erase", ] start_download_data(get_args(args)) assert dl_mock.call_count == 4 + assert dl_mock.call_args[1]['timerange'].starttype is None + assert dl_mock.call_args[1]['timerange'].stoptype is None assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog) assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) +def test_download_data_days(mocker, markets, caplog): + dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch.object(Path, "unlink", MagicMock()) + + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20", + ] + + start_download_data(get_args(args)) + + assert dl_mock.call_count == 4 + assert dl_mock.call_args[1]['timerange'].starttype == 'date' + + assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) + + def test_download_data_no_markets(mocker, caplog): dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) patch_exchange(mocker) @@ -76,10 +104,28 @@ def test_download_data_no_markets(mocker, caplog): args = [ "download-data", "--exchange", "binance", - "--pairs", "ETH/BTC", "XRP/BTC" + "--pairs", "ETH/BTC", "XRP/BTC", ] start_download_data(get_args(args)) assert dl_mock.call_count == 0 assert log_has("Skipping pair ETH/BTC...", caplog) assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) + + +def test_download_data_keyboardInterrupt(mocker, caplog, markets): + dl_mock = mocker.patch('freqtrade.utils.download_pair_history', + MagicMock(side_effect=KeyboardInterrupt)) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + ] + with pytest.raises(SystemExit): + start_download_data(get_args(args)) + + assert dl_mock.call_count == 1 From 7a79b292e46dfab4e19e4635c8720bac8cde7676 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Aug 2019 07:05:42 +0200 Subject: [PATCH 23/42] Fix bug in pairs fallback resolving --- freqtrade/configuration/configuration.py | 3 ++- freqtrade/tests/test_configuration.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index c51153f4b..676b0c594 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -380,7 +380,7 @@ class Configuration(object): config['pairs'].sort() return - if "config" in self.args: + if "config" in self.args and self.args.config: logger.info("Using pairlist from configuration.") config['pairs'] = config.get('exchange', {}).get('pair_whitelist') else: @@ -388,3 +388,4 @@ class Configuration(object): pairs_file = Path(config['datadir']) / "pairs.json" if pairs_file.exists(): config['pairs'] = json_load(pairs_file) + config['pairs'].sort() diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index c351b9b72..b6e8a76d9 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -792,3 +792,21 @@ def test_pairlist_resolving_with_config_pl_not_exists(mocker, default_conf): with pytest.raises(OperationalException, match=r"No pairs file found with path.*"): configuration = Configuration(args) configuration.get_config() + + +def test_pairlist_resolving_fallback(mocker): + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch("freqtrade.configuration.configuration.json_load", + MagicMock(return_value=['XRP/BTC', 'ETH/BTC'])) + arglist = [ + 'download-data', + '--exchange', 'binance' + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] + assert config['exchange']['name'] == 'binance' From 08fa5136e11843c7ea3da2c464f1a0e77d921f03 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Aug 2019 07:19:46 +0200 Subject: [PATCH 24/42] use copy of minimal_config ... --- freqtrade/configuration/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 676b0c594..75319ac47 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -54,7 +54,7 @@ class Configuration(object): config: Dict[str, Any] = {} if not files: - return constants.MINIMAL_CONFIG + return constants.MINIMAL_CONFIG.copy() # We expect here a list of config filenames for path in files: From 5e440a4cdc6baa4ecadcc37593f373fe06aeb257 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Aug 2019 06:55:19 +0200 Subject: [PATCH 25/42] Improve docs to point to `freqtrade download-data` --- docs/backtesting.md | 2 +- docs/bot-usage.md | 16 ++++------------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index f666c5b49..543422fee 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -3,7 +3,7 @@ This page explains how to validate your strategy performance by using Backtesting. -## Getting data for backtesting / hyperopt +## Getting data for backtesting and hyperopt To download backtesting data (candles / OHLCV), we recommend using the `freqtrade download-data` command. diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 31c5812ad..2873f5e8f 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -184,19 +184,11 @@ optional arguments: result.json) ``` -### How to use **--refresh-pairs-cached** parameter? +### Getting historic data for backtesting -The first time your run Backtesting, it will take the pairs you have -set in your config file and download data from the Exchange. - -If for any reason you want to update your data set, you use -`--refresh-pairs-cached` to force Backtesting to update the data it has. - -!!! Note - Use it only if you want to update your data set. You will not be able to come back to the previous version. - -To test your strategy with latest data, we recommend continuing using -the parameter `-l` or `--live`. +The first time your run Backtesting, you will need to download some historic data first. +This can be accomplished by using `freqtrade download-data`. +Check the corresponding [help page section](backtesting.md#Getting-data-for-backtesting-and-hyperopt) for more details ## Hyperopt commands From 8e96ac876597b686e66df3e4a238f324a9be95d8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Aug 2019 15:45:30 +0200 Subject: [PATCH 26/42] Split exception tests for create_order --- freqtrade/tests/exchange/test_exchange.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 6b833054d..123f1fcfd 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -779,7 +779,13 @@ def test_sell_prod(default_conf, mocker, exchange_name): with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + exchange.sell(pair='ETH/BTC', ordertype='limit', amount=1, rate=200) + + # Market orders don't require price, so the behaviour is slightly different + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + exchange.sell(pair='ETH/BTC', ordertype='market', amount=1, rate=200) with pytest.raises(TemporaryError): api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No Connection")) From ee7ba96e8524558d8d7ed068fb939d18aac0a711 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Aug 2019 15:46:10 +0200 Subject: [PATCH 27/42] Don't do calculations in exception handlers when one element can be None fixes #2011 --- freqtrade/exchange/exchange.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 5119e0fcd..67a79178f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -408,12 +408,12 @@ class Exchange(object): except ccxt.InsufficientFunds as e: raise DependencyException( f'Insufficient funds to create {ordertype} {side} order on market {pair}.' - f'Tried to {side} amount {amount} at rate {rate} (total {rate * amount}).' + f'Tried to {side} amount {amount} at rate {rate}.' f'Message: {e}') from e except ccxt.InvalidOrder as e: raise DependencyException( f'Could not create {ordertype} {side} order on market {pair}.' - f'Tried to {side} amount {amount} at rate {rate} (total {rate * amount}).' + f'Tried to {side} amount {amount} at rate {rate}.' f'Message: {e}') from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( From 045ac1019e37359bf6d04f9989af50493122b434 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Aug 2019 15:48:20 +0200 Subject: [PATCH 28/42] Split test for buy-orders too --- freqtrade/exchange/exchange.py | 2 +- freqtrade/tests/exchange/test_exchange.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 67a79178f..6e281b9b2 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -472,7 +472,7 @@ class Exchange(object): order = self.create_order(pair, ordertype, 'sell', amount, rate, params) logger.info('stoploss limit order added for %s. ' - 'stop price: %s. limit: %s' % (pair, stop_price, rate)) + 'stop price: %s. limit: %s', pair, stop_price, rate) return order @retrier diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 123f1fcfd..397326258 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -656,7 +656,13 @@ def test_buy_prod(default_conf, mocker, exchange_name): with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.buy(pair='ETH/BTC', ordertype=order_type, + exchange.buy(pair='ETH/BTC', ordertype='limit', + amount=1, rate=200, time_in_force=time_in_force) + + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + exchange.buy(pair='ETH/BTC', ordertype='market', amount=1, rate=200, time_in_force=time_in_force) with pytest.raises(TemporaryError): From ddfadbb69ee2bf18191d94d053d3ef2301286030 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Aug 2019 16:10:10 +0200 Subject: [PATCH 29/42] Validate configuration consistency after loading strategy --- freqtrade/configuration/__init__.py | 1 + .../{json_schema.py => config_validation.py} | 35 ++++++++++++++++- freqtrade/configuration/configuration.py | 38 +++---------------- freqtrade/freqtradebot.py | 2 + freqtrade/tests/test_configuration.py | 14 +++---- 5 files changed, 47 insertions(+), 43 deletions(-) rename freqtrade/configuration/{json_schema.py => config_validation.py} (53%) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 7b476d173..ac59421a7 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,3 +1,4 @@ from freqtrade.configuration.arguments import Arguments # noqa: F401 from freqtrade.configuration.timerange import TimeRange # noqa: F401 from freqtrade.configuration.configuration import Configuration # noqa: F401 +from freqtrade.configuration.config_validation import validate_config_consistency # noqa: F401 diff --git a/freqtrade/configuration/json_schema.py b/freqtrade/configuration/config_validation.py similarity index 53% rename from freqtrade/configuration/json_schema.py rename to freqtrade/configuration/config_validation.py index 4c6f4a4a0..bda60f90b 100644 --- a/freqtrade/configuration/json_schema.py +++ b/freqtrade/configuration/config_validation.py @@ -4,7 +4,7 @@ from typing import Any, Dict from jsonschema import Draft4Validator, validators from jsonschema.exceptions import ValidationError, best_match -from freqtrade import constants +from freqtrade import constants, OperationalException logger = logging.getLogger(__name__) @@ -51,3 +51,36 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: raise ValidationError( best_match(Draft4Validator(constants.CONF_SCHEMA).iter_errors(conf)).message ) + + +def validate_config_consistency(conf: Dict[str, Any]) -> None: + """ + Validate the configuration consistency. + Should be ran after loading both configuration and strategy, + since strategies can set certain configuration settings too. + :param conf: Config in JSON format + :return: Returns None if everything is ok, otherwise throw an OperationalException + """ + # validating trailing stoploss + _validate_trailing_stoploss(conf) + + +def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None: + + # Skip if trailing stoploss is not activated + if not conf.get('trailing_stop', False): + return + + tsl_positive = float(conf.get('trailing_stop_positive', 0)) + tsl_offset = float(conf.get('trailing_stop_positive_offset', 0)) + tsl_only_offset = conf.get('trailing_only_offset_is_reached', False) + + if tsl_only_offset: + if tsl_positive == 0.0: + raise OperationalException( + f'The config trailing_only_offset_is_reached needs ' + 'trailing_stop_positive_offset to be more than 0 in your config.') + if tsl_positive > 0 and 0 < tsl_offset <= tsl_positive: + raise OperationalException( + f'The config trailing_stop_positive_offset needs ' + 'to be greater than trailing_stop_positive_offset in your config.') diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index c95246fc0..486857153 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -6,10 +6,11 @@ import warnings from argparse import Namespace from typing import Any, Callable, Dict, List, Optional -from freqtrade import OperationalException, constants +from freqtrade import constants from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.create_datadir import create_datadir -from freqtrade.configuration.json_schema import validate_config_schema +from freqtrade.configuration.config_validation import (validate_config_schema, + validate_config_consistency) from freqtrade.configuration.load_config import load_config_file from freqtrade.loggers import setup_logging from freqtrade.misc import deep_merge_dicts @@ -77,8 +78,6 @@ class Configuration(object): # Load all configs config: Dict[str, Any] = Configuration.from_files(self.args.config) - self._validate_config_consistency(config) - self._process_common_options(config) self._process_optimize_options(config) @@ -87,6 +86,8 @@ class Configuration(object): self._process_runmode(config) + validate_config_consistency(config) + return config def _process_logging_options(self, config: Dict[str, Any]) -> None: @@ -285,35 +286,6 @@ class Configuration(object): config.update({'runmode': self.runmode}) - def _validate_config_consistency(self, conf: Dict[str, Any]) -> None: - """ - Validate the configuration consistency - :param conf: Config in JSON format - :return: Returns None if everything is ok, otherwise throw an OperationalException - """ - # validating trailing stoploss - self._validate_trailing_stoploss(conf) - - def _validate_trailing_stoploss(self, conf: Dict[str, Any]) -> None: - - # Skip if trailing stoploss is not activated - if not conf.get('trailing_stop', False): - return - - tsl_positive = float(conf.get('trailing_stop_positive', 0)) - tsl_offset = float(conf.get('trailing_stop_positive_offset', 0)) - tsl_only_offset = conf.get('trailing_only_offset_is_reached', False) - - if tsl_only_offset: - if tsl_positive == 0.0: - raise OperationalException( - f'The config trailing_only_offset_is_reached needs ' - 'trailing_stop_positive_offset to be more than 0 in your config.') - if tsl_positive > 0 and 0 < tsl_offset <= tsl_positive: - raise OperationalException( - f'The config trailing_stop_positive_offset needs ' - 'to be greater than trailing_stop_positive_offset in your config.') - def _args_to_config(self, config: Dict[str, Any], argname: str, logstring: str, logfun: Optional[Callable] = None, deprecated_msg: Optional[str] = None) -> None: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 68b45d96f..af29604f5 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -16,6 +16,7 @@ from freqtrade import (DependencyException, OperationalException, InvalidOrderEx from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge +from freqtrade.configuration import validate_config_consistency from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType @@ -50,6 +51,7 @@ class FreqtradeBot(object): self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy + validate_config_consistency(config) self.rpc: RPCManager = RPCManager(self) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 8cbd02ece..3a0077ef2 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -2,7 +2,6 @@ import json import logging import warnings -from argparse import Namespace from copy import deepcopy from pathlib import Path from unittest.mock import MagicMock @@ -11,10 +10,10 @@ import pytest from jsonschema import Draft4Validator, ValidationError, validate from freqtrade import OperationalException, constants -from freqtrade.configuration import Arguments, Configuration +from freqtrade.configuration import Arguments, Configuration, validate_config_consistency from freqtrade.configuration.check_exchange import check_exchange +from freqtrade.configuration.config_validation import validate_config_schema from freqtrade.configuration.create_datadir import create_datadir -from freqtrade.configuration.json_schema import validate_config_schema from freqtrade.configuration.load_config import load_config_file from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.loggers import _set_loggers @@ -625,21 +624,18 @@ def test_validate_tsl(default_conf): with pytest.raises(OperationalException, match=r'The config trailing_only_offset_is_reached needs ' 'trailing_stop_positive_offset to be more than 0 in your config.'): - configuration = Configuration(Namespace()) - configuration._validate_config_consistency(default_conf) + validate_config_consistency(default_conf) default_conf['trailing_stop_positive_offset'] = 0.01 default_conf['trailing_stop_positive'] = 0.015 with pytest.raises(OperationalException, match=r'The config trailing_stop_positive_offset needs ' 'to be greater than trailing_stop_positive_offset in your config.'): - configuration = Configuration(Namespace()) - configuration._validate_config_consistency(default_conf) + validate_config_consistency(default_conf) default_conf['trailing_stop_positive'] = 0.01 default_conf['trailing_stop_positive_offset'] = 0.015 - Configuration(Namespace()) - configuration._validate_config_consistency(default_conf) + validate_config_consistency(default_conf) def test_load_config_test_comments() -> None: From 611850bf91a343d056259f3bb6ab7bc9b683c59f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Aug 2019 16:19:24 +0200 Subject: [PATCH 30/42] Add edge/dynamic_whitelist validation --- freqtrade/configuration/config_validation.py | 16 ++++++++++++++++ freqtrade/tests/test_configuration.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index bda60f90b..92846b704 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -63,6 +63,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None: """ # validating trailing stoploss _validate_trailing_stoploss(conf) + _validate_edge(conf) def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None: @@ -84,3 +85,18 @@ def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None: raise OperationalException( f'The config trailing_stop_positive_offset needs ' 'to be greater than trailing_stop_positive_offset in your config.') + + +def _validate_edge(conf: Dict[str, Any]) -> None: + """ + Edge and Dynamic whitelist should not both be enabled, since edge overrides dynamic whitelists. + """ + + if not conf.get('edge', {}).get('enabled'): + return + + if conf.get('pairlist', {}).get('method') == 'VolumePairList': + raise OperationalException( + "Edge and VolumePairList are incompatible, " + "Edge will override whatever pairs VolumePairlist selects." + ) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 3a0077ef2..19d2a28ee 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -638,6 +638,22 @@ def test_validate_tsl(default_conf): validate_config_consistency(default_conf) +def test_validate_edge(edge_conf): + edge_conf.update({"pairlist": { + "method": "VolumePairList", + }}) + + with pytest.raises(OperationalException, + match="Edge and VolumePairList are incompatible, " + "Edge will override whatever pairs VolumePairlist selects."): + validate_config_consistency(edge_conf) + + edge_conf.update({"pairlist": { + "method": "StaticPairList", + }}) + validate_config_consistency(edge_conf) + + def test_load_config_test_comments() -> None: """ Load config with comments From b6462cd51f707f4160fa77d8bf2eb36c73a7695e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Aug 2019 16:22:18 +0200 Subject: [PATCH 31/42] Add explaining comment --- freqtrade/freqtradebot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index af29604f5..e5ecef8bf 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -51,6 +51,8 @@ class FreqtradeBot(object): self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy + + # Check config consistency here since strategies can set certain options validate_config_consistency(config) self.rpc: RPCManager = RPCManager(self) From d785d7637085edcaf382a26df2d71cb8612f6a7e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Aug 2019 18:06:36 +0200 Subject: [PATCH 32/42] make VolumePairlist less verbose no need to print the full whitelist on every iteration --- freqtrade/constants.py | 1 - freqtrade/pairlist/VolumePairList.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 9b73adcfe..bd3ba21fd 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -5,7 +5,6 @@ bot constants """ DEFAULT_CONFIG = 'config.json' DEFAULT_EXCHANGE = 'bittrex' -DYNAMIC_WHITELIST = 20 # pairs PROCESS_THROTTLE_SECS = 5 # sec DEFAULT_TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 9a2e2eac4..b9b7977ab 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -55,7 +55,6 @@ class VolumePairList(IPairList): # Generate dynamic whitelist self._whitelist = self._gen_pair_whitelist( self._config['stake_currency'], self._sort_key)[:self._number_pairs] - logger.info(f"Searching pairs: {self._whitelist}") @cached(TTLCache(maxsize=1, ttl=1800)) def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: @@ -92,4 +91,6 @@ class VolumePairList(IPairList): valid_tickers.remove(t) pairs = [s['symbol'] for s in valid_tickers] + logger.info(f"Searching pairs: {self._whitelist}") + return pairs From ea4db0ffb6dd0007b4df471a3e3f8f90d9e3599d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Aug 2019 18:06:57 +0200 Subject: [PATCH 33/42] Pass object-name to loader to fix logging --- freqtrade/resolvers/iresolver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 841c3cf43..310c54015 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -29,7 +29,8 @@ class IResolver(object): """ # Generate spec based on absolute path - spec = importlib.util.spec_from_file_location('unknown', str(module_path)) + # Pass object_name as first argument to have logging print a reasonable name. + spec = importlib.util.spec_from_file_location(object_name, str(module_path)) module = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(module) # type: ignore # importlib does not use typehints From a4ede02cedd22ddc1147f1fcffd224b554af970d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Aug 2019 19:38:23 +0200 Subject: [PATCH 34/42] Gracefully handle problems with dry-run orders --- freqtrade/exchange/exchange.py | 9 +++++++-- freqtrade/tests/exchange/test_exchange.py | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 5119e0fcd..7ae40381d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -696,8 +696,13 @@ class Exchange(object): @retrier def get_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - order = self._dry_run_open_orders[order_id] - return order + try: + order = self._dry_run_open_orders[order_id] + return order + except KeyError as e: + # Gracefully handle errors with dry-run orders. + raise InvalidOrderException( + f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e try: return self._api.fetch_order(order_id, pair) except ccxt.InvalidOrder as e: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 6b833054d..6a7dfa04b 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1328,6 +1328,9 @@ def test_get_order(default_conf, mocker, exchange_name): print(exchange.get_order('X', 'TKN/BTC')) assert exchange.get_order('X', 'TKN/BTC').myid == 123 + with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'): + exchange.get_order('Y', 'TKN/BTC') + default_conf['dry_run'] = False api_mock = MagicMock() api_mock.fetch_order = MagicMock(return_value=456) From 9ad9ce0da1b46a0822cba4f100257a177211530f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2019 10:52:53 +0000 Subject: [PATCH 35/42] Bump ccxt from 1.18.1063 to 1.18.1068 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.18.1063 to 1.18.1068. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/1.18.1063...1.18.1068) Signed-off-by: dependabot-preview[bot] --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 4666fc053..3d80c3ef5 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.1063 +ccxt==1.18.1068 SQLAlchemy==1.3.7 python-telegram-bot==11.1.0 arrow==0.14.5 From 70b1a05d976aad8a7c808ba7ee8a27156081b7b8 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 20 Aug 2019 01:32:02 +0300 Subject: [PATCH 36/42] example in the docs changed --- docs/strategy-customization.md | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 0d08bdd02..d71ebfded 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -274,27 +274,24 @@ Please always check the mode of operation to select the correct method to get da #### Possible options for DataProvider -- `available_pairs` - Property with tuples listing cached pairs with their intervals. (pair, interval) -- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for all pairs in the whitelist, returns DataFrame or empty DataFrame -- `historic_ohlcv(pair, ticker_interval)` - Data stored on disk +- `available_pairs` - Property with tuples listing cached pairs with their intervals (pair, interval). +- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for the pair, returns DataFrame or empty DataFrame. +- `historic_ohlcv(pair, ticker_interval)` - Returns historical data stored on disk. +- `get_pair_dataframe(pair, ticker_interval)` - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). - `runmode` - Property containing the current runmode. -#### ohlcv / historic_ohlcv +#### Example: fetch live ohlcv / historic data for the first informative pair ``` python if self.dp: - if self.dp.runmode in ('live', 'dry_run'): - if (f'{self.stake_currency}/BTC', self.ticker_interval) in self.dp.available_pairs: - data_eth = self.dp.ohlcv(pair='{self.stake_currency}/BTC', - ticker_interval=self.ticker_interval) - else: - # Get historic ohlcv data (cached on disk). - history_eth = self.dp.historic_ohlcv(pair='{self.stake_currency}/BTC', - ticker_interval='1h') + inf_pair, inf_timeframe = self.informative_pairs()[0] + informative = self.dp.get_pair_dataframe(pair=inf_pair, + ticker_interval=inf_timeframe) ``` !!! Warning Warning about backtesting - Be carefull when using dataprovider in backtesting. `historic_ohlcv()` provides the full time-range in one go, + Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` + for the backtesting runmode) provides the full time-range in one go, so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). !!! Warning Warning in hyperopt From 8d1a575a9b975e1753d312fbf72d8e4f640354f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Aug 2019 06:39:28 +0200 Subject: [PATCH 37/42] Reword documentation --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 543422fee..3712fddce 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -5,7 +5,7 @@ Backtesting. ## Getting data for backtesting and hyperopt -To download backtesting data (candles / OHLCV), we recommend using the `freqtrade download-data` command. +To download backtesting data (candles / OHLCV) and hyperoptimization using the `freqtrade download-data` command. If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes. Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. From be308ff91428d8b4b53cc3f5a199289d5652279e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Aug 2019 09:45:28 +0200 Subject: [PATCH 38/42] Fix grammar error in documentation --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 3712fddce..48f658ab9 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -5,7 +5,7 @@ Backtesting. ## Getting data for backtesting and hyperopt -To download backtesting data (candles / OHLCV) and hyperoptimization using the `freqtrade download-data` command. +To download backtesting data (candles / OHLCV) and hyperoptimization use the `freqtrade download-data` command. If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes. Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. From 210f66e48bbbd654f61d566eca9224121c9ef147 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Aug 2019 19:34:18 +0200 Subject: [PATCH 39/42] Improve wording --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 48f658ab9..90256283e 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -5,7 +5,7 @@ Backtesting. ## Getting data for backtesting and hyperopt -To download backtesting data (candles / OHLCV) and hyperoptimization use the `freqtrade download-data` command. +To download data (candles / OHLCV) needed for backtesting and hyperoptimization use the `freqtrade download-data` command. If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes. Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. From 14aaf8976f6aefc48f99018cc7bc732339151f74 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 21 Aug 2019 02:26:58 +0300 Subject: [PATCH 40/42] fix download replacement script --- scripts/download_backtest_data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 496f83c7d..a8f919a10 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -1,8 +1,10 @@ +#!/usr/bin/env python3 + import sys print("This script has been integrated into freqtrade " - "and it's functionality is available by calling `freqtrade download-data`.") + "and its functionality is available by calling `freqtrade download-data`.") print("Please check the documentation on https://www.freqtrade.io/en/latest/backtesting/ " "for details.") From 75b2db4424dc40e1c9dccfb582479a08bf1aa964 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 21 Aug 2019 06:58:56 +0200 Subject: [PATCH 41/42] FIx loading pairs-list --- freqtrade/configuration/configuration.py | 16 +++++++++------- freqtrade/misc.py | 3 ++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 329058cef..b3f54d59e 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -346,10 +346,9 @@ class Configuration(object): # or if pairs file is specified explicitely if not pairs_file.exists(): raise OperationalException(f'No pairs file found with path "{pairs_file}".') - - config['pairs'] = json_load(pairs_file) - - config['pairs'].sort() + with pairs_file.open('r') as f: + config['pairs'] = json_load(f) + config['pairs'].sort() return if "config" in self.args and self.args.config: @@ -357,7 +356,10 @@ class Configuration(object): config['pairs'] = config.get('exchange', {}).get('pair_whitelist') else: # Fall back to /dl_path/pairs.json - pairs_file = Path(config['datadir']) / "pairs.json" + pairs_file = Path(config['datadir']) / config['exchange']['name'].lower() / "pairs.json" + print(config['datadir']) if pairs_file.exists(): - config['pairs'] = json_load(pairs_file) - config['pairs'].sort() + with pairs_file.open('r') as f: + config['pairs'] = json_load(f) + if 'pairs' in config: + config['pairs'].sort() diff --git a/freqtrade/misc.py b/freqtrade/misc.py index d01d6a254..12a90a14d 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -6,6 +6,7 @@ import logging import re from datetime import datetime from pathlib import Path +from typing.io import IO import numpy as np import rapidjson @@ -60,7 +61,7 @@ def file_dump_json(filename: Path, data, is_zip=False) -> None: logger.debug(f'done json to "{filename}"') -def json_load(datafile): +def json_load(datafile: IO): """ load data with rapidjson Use this to have a consistent experience, From 13ffb392457269bee56977dbc73d70752b76d531 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 21 Aug 2019 06:59:07 +0200 Subject: [PATCH 42/42] Adjust tests to fixed loading method --- freqtrade/tests/test_configuration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 5cfee0698..fc3adad44 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -770,6 +770,7 @@ def test_pairlist_resolving_with_config_pl(mocker, default_conf): load_mock = mocker.patch("freqtrade.configuration.configuration.json_load", MagicMock(return_value=['XRP/BTC', 'ETH/BTC'])) mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch.object(Path, "open", MagicMock(return_value=MagicMock())) arglist = [ '--config', 'config.json', @@ -808,6 +809,7 @@ def test_pairlist_resolving_with_config_pl_not_exists(mocker, default_conf): def test_pairlist_resolving_fallback(mocker): mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch.object(Path, "open", MagicMock(return_value=MagicMock())) mocker.patch("freqtrade.configuration.configuration.json_load", MagicMock(return_value=['XRP/BTC', 'ETH/BTC'])) arglist = [