From 2af9ffa7f2cf3f7bdfce4a4b582df667d5bd6cc5 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 16 Dec 2019 21:43:33 +0300 Subject: [PATCH 1/5] Align refresh_backtest_ to each other --- freqtrade/data/history.py | 16 ++++++++-------- freqtrade/utils.py | 2 +- tests/data/test_history.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index ddcae89ab..03aaeb0cc 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -334,12 +334,12 @@ def download_pair_history(datadir: Path, def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], - dl_path: Path, timerange: Optional[TimeRange] = None, + datadir: Path, timerange: Optional[TimeRange] = None, erase=False) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. - Used by freqtrade download-data - :return: Pairs not available + Used by freqtrade download-data subcommand. + :return: List of pairs that are not available. """ pairs_not_available = [] for pair in pairs: @@ -349,14 +349,14 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes continue for timeframe in timeframes: - dl_file = pair_data_filename(dl_path, pair, timeframe) + dl_file = pair_data_filename(datadir, pair, timeframe) if erase and dl_file.exists(): logger.info( f'Deleting existing data for pair {pair}, interval {timeframe}.') dl_file.unlink() logger.info(f'Downloading pair {pair}, interval {timeframe}.') - download_pair_history(datadir=dl_path, exchange=exchange, + download_pair_history(datadir=datadir, exchange=exchange, pair=pair, timeframe=str(timeframe), timerange=timerange) return pairs_not_available @@ -407,9 +407,9 @@ def download_trades_history(datadir: Path, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, timerange: TimeRange, erase=False) -> List[str]: """ - Refresh stored trades data. - Used by freqtrade download-data - :return: Pairs not available + Refresh stored trades data for backtesting and hyperopt operations. + Used by freqtrade download-data subcommand. + :return: List of pairs that are not available. """ pairs_not_available = [] for pair in pairs: diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 230fcf268..9e01c7ea6 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -213,7 +213,7 @@ def start_download_data(args: Dict[str, Any]) -> None: else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=config["pairs"], timeframes=config["timeframes"], - dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) + datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 6d89ab7c5..586492703 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -580,7 +580,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test ex = get_patched_exchange(mocker, default_conf) timerange = TimeRange.parse_timerange("20190101-20190102") refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], - timeframes=["1m", "5m"], dl_path=testdatadir, + timeframes=["1m", "5m"], datadir=testdatadir, timerange=timerange, erase=True ) @@ -600,7 +600,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): timerange = TimeRange.parse_timerange("20190101-20190102") unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["BTT/BTC", "LTC/USDT"], timeframes=["1m", "5m"], - dl_path=testdatadir, + datadir=testdatadir, timerange=timerange, erase=False ) From 4cd45b6535d4ed613718c7c5656701a81891c1ee Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 16 Dec 2019 21:57:03 +0300 Subject: [PATCH 2/5] Rename download_*_history as non-public --- freqtrade/data/history.py | 40 ++++++++++++++--------------- tests/data/test_history.py | 52 +++++++++++++++++++------------------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 03aaeb0cc..15c5ed451 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -155,11 +155,11 @@ def load_pair_history(pair: str, # The user forced the refresh of pairs if refresh_pairs: - download_pair_history(datadir=datadir, - exchange=exchange, - pair=pair, - timeframe=timeframe, - timerange=timerange) + _download_pair_history(datadir=datadir, + exchange=exchange, + pair=pair, + timeframe=timeframe, + timerange=timerange) pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup) @@ -277,11 +277,11 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str, return (data, since_ms) -def download_pair_history(datadir: Path, - exchange: Optional[Exchange], - pair: str, - timeframe: str = '5m', - timerange: Optional[TimeRange] = None) -> bool: +def _download_pair_history(datadir: Path, + exchange: Optional[Exchange], + pair: str, + timeframe: str = '5m', + timerange: Optional[TimeRange] = None) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that @@ -356,16 +356,16 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes dl_file.unlink() logger.info(f'Downloading pair {pair}, interval {timeframe}.') - download_pair_history(datadir=datadir, exchange=exchange, - pair=pair, timeframe=str(timeframe), - timerange=timerange) + _download_pair_history(datadir=datadir, exchange=exchange, + pair=pair, timeframe=str(timeframe), + timerange=timerange) return pairs_not_available -def download_trades_history(datadir: Path, - exchange: Exchange, - pair: str, - timerange: Optional[TimeRange] = None) -> bool: +def _download_trades_history(datadir: Path, + exchange: Exchange, + pair: str, + timerange: Optional[TimeRange] = None) -> bool: """ Download trade history from the exchange. Appends to previously downloaded trades data. @@ -425,9 +425,9 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: dl_file.unlink() logger.info(f'Downloading trades for pair {pair}.') - download_trades_history(datadir=datadir, exchange=exchange, - pair=pair, - timerange=timerange) + _download_trades_history(datadir=datadir, exchange=exchange, + pair=pair, + timerange=timerange) return pairs_not_available diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 586492703..c304aa97d 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -14,9 +14,9 @@ from freqtrade import OperationalException from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.history import (_load_cached_data_for_updating, + _download_pair_history, + _download_trades_history, convert_trades_to_ohlcv, - download_pair_history, - download_trades_history, load_tickerdata_file, pair_data_filename, pair_trades_filename, refresh_backtest_ohlcv_data, @@ -267,12 +267,12 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda assert not file1_1.is_file() assert not file2_1.is_file() - assert download_pair_history(datadir=testdatadir, exchange=exchange, - pair='MEME/BTC', - timeframe='1m') - assert download_pair_history(datadir=testdatadir, exchange=exchange, - pair='CFI/BTC', - timeframe='1m') + assert _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='MEME/BTC', + timeframe='1m') + assert _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='CFI/BTC', + timeframe='1m') assert not exchange._pairs_last_refresh_time assert file1_1.is_file() assert file2_1.is_file() @@ -284,12 +284,12 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda assert not file1_5.is_file() assert not file2_5.is_file() - assert download_pair_history(datadir=testdatadir, exchange=exchange, - pair='MEME/BTC', - timeframe='5m') - assert download_pair_history(datadir=testdatadir, exchange=exchange, - pair='CFI/BTC', - timeframe='5m') + assert _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='MEME/BTC', + timeframe='5m') + assert _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='CFI/BTC', + timeframe='5m') assert not exchange._pairs_last_refresh_time assert file1_5.is_file() assert file2_5.is_file() @@ -307,8 +307,8 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) - download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') - download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m') + _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') + _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m') assert json_dump_mock.call_count == 2 @@ -324,9 +324,9 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, _backup_file(file1_1) _backup_file(file1_5) - assert not download_pair_history(datadir=testdatadir, exchange=exchange, - pair='MEME/BTC', - timeframe='1m') + assert not _download_pair_history(datadir=testdatadir, exchange=exchange, + pair='MEME/BTC', + timeframe='1m') # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file1_5) @@ -570,7 +570,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) @@ -591,7 +591,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock()) ex = get_patched_exchange(mocker, default_conf) mocker.patch( @@ -611,7 +611,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history.download_trades_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history._download_trades_history', MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) @@ -646,8 +646,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert not file1.is_file() - assert download_trades_history(datadir=testdatadir, exchange=exchange, - pair='ETH/BTC') + assert _download_trades_history(datadir=testdatadir, exchange=exchange, + pair='ETH/BTC') assert log_has("New Amount of trades: 5", caplog) assert file1.is_file() @@ -657,8 +657,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', MagicMock(side_effect=ValueError)) - assert not download_trades_history(datadir=testdatadir, exchange=exchange, - pair='ETH/BTC') + assert not _download_trades_history(datadir=testdatadir, exchange=exchange, + pair='ETH/BTC') assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) From fa968996ed980df63aa139180af0ec5799b74b26 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 16 Dec 2019 22:01:26 +0300 Subject: [PATCH 3/5] Remove useless check --- freqtrade/data/history.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 15c5ed451..5176a9b19 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -295,11 +295,6 @@ def _download_pair_history(datadir: Path, :param timerange: range of time to download :return: bool with success state """ - if not exchange: - raise OperationalException( - "Exchange needs to be initialized when downloading pair history data" - ) - try: logger.info( f'Download history data for pair: "{pair}", timeframe: {timeframe} ' From a6fc743d854e33ddd44fb11cea277ab1a6482f86 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 16 Dec 2019 22:12:26 +0300 Subject: [PATCH 4/5] Align code in _download_*_history() --- freqtrade/data/history.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 5176a9b19..79500c512 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -307,11 +307,12 @@ def _download_pair_history(datadir: Path, logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') # Default since_ms to 30 days if nothing is given - new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms if since_ms - else + new_data = exchange.get_historic_ohlcv(pair=pair, + timeframe=timeframe, + since_ms=since_ms if since_ms else int(arrow.utcnow().shift( - days=-30).float_timestamp) * 1000) + days=-30).float_timestamp) * 1000 + ) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) @@ -376,11 +377,11 @@ def _download_trades_history(datadir: Path, logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None') logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None') + # Default since_ms to 30 days if nothing is given new_trades = exchange.get_historic_trades(pair=pair, since=since if since else int(arrow.utcnow().shift( days=-30).float_timestamp) * 1000, - # until=xxx, from_id=from_id, ) trades.extend(new_trades[1]) From 0277cd82ea66cd019138f0f45caaeed3b0afc187 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 16 Dec 2019 23:25:57 +0300 Subject: [PATCH 5/5] Make mypy happy --- freqtrade/data/history.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 79500c512..2c01f9f48 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -295,6 +295,11 @@ def _download_pair_history(datadir: Path, :param timerange: range of time to download :return: bool with success state """ + if not exchange: + raise OperationalException( + "Exchange needs to be initialized when downloading pair history data" + ) + try: logger.info( f'Download history data for pair: "{pair}", timeframe: {timeframe} '