adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2672 from hroff-1902/minor-data-history-2

Browse Source 
Minor: improvements in data/history.py
This commit is contained in:
Matthias 2019-12-17 09:22:56 +01:00 committed by GitHub
commit 1042f9847a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 62 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
freqtrade/data/history.py
View File

@ -155,11 +155,11 @@ def load_pair_history(pair: str,
# The user forced the refresh of pairs # The user forced the refresh of pairs
if refresh_pairs: if refresh_pairs:
download_pair_history(datadir=datadir, _download_pair_history(datadir=datadir,
exchange=exchange, exchange=exchange,
pair=pair, pair=pair,
timeframe=timeframe, timeframe=timeframe,
timerange=timerange) timerange=timerange)
pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup) pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup)
@ -277,11 +277,11 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str,
return (data, since_ms) return (data, since_ms)
def download_pair_history(datadir: Path, def _download_pair_history(datadir: Path,
exchange: Optional[Exchange], exchange: Optional[Exchange],
pair: str, pair: str,
timeframe: str = '5m', timeframe: str = '5m',
timerange: Optional[TimeRange] = None) -> bool: timerange: Optional[TimeRange] = None) -> bool:
""" """
Download latest candles from the exchange for the pair and timeframe passed in parameters Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that The data is downloaded starting from the last correct data that
@ -312,11 +312,12 @@ def download_pair_history(datadir: Path,
logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
# Default since_ms to 30 days if nothing is given # Default since_ms to 30 days if nothing is given
new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, new_data = exchange.get_historic_ohlcv(pair=pair,
since_ms=since_ms if since_ms timeframe=timeframe,
else since_ms=since_ms if since_ms else
int(arrow.utcnow().shift( int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000) days=-30).float_timestamp) * 1000
)
data.extend(new_data) data.extend(new_data)
logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))
@ -334,12 +335,12 @@ def download_pair_history(datadir: Path,
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
dl_path: Path, timerange: Optional[TimeRange] = None, datadir: Path, timerange: Optional[TimeRange] = None,
erase=False) -> List[str]: erase=False) -> List[str]:
""" """
Refresh stored ohlcv data for backtesting and hyperopt operations. Refresh stored ohlcv data for backtesting and hyperopt operations.
Used by freqtrade download-data Used by freqtrade download-data subcommand.
:return: Pairs not available :return: List of pairs that are not available.
""" """
pairs_not_available = [] pairs_not_available = []
for pair in pairs: for pair in pairs:
@ -349,23 +350,23 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
continue continue
for timeframe in timeframes: for timeframe in timeframes:
dl_file = pair_data_filename(dl_path, pair, timeframe) dl_file = pair_data_filename(datadir, pair, timeframe)
if erase and dl_file.exists(): if erase and dl_file.exists():
logger.info( logger.info(
f'Deleting existing data for pair {pair}, interval {timeframe}.') f'Deleting existing data for pair {pair}, interval {timeframe}.')
dl_file.unlink() dl_file.unlink()
logger.info(f'Downloading pair {pair}, interval {timeframe}.') logger.info(f'Downloading pair {pair}, interval {timeframe}.')
download_pair_history(datadir=dl_path, exchange=exchange, _download_pair_history(datadir=datadir, exchange=exchange,
pair=pair, timeframe=str(timeframe), pair=pair, timeframe=str(timeframe),
timerange=timerange) timerange=timerange)
return pairs_not_available return pairs_not_available
def download_trades_history(datadir: Path, def _download_trades_history(datadir: Path,
exchange: Exchange, exchange: Exchange,
pair: str, pair: str,
timerange: Optional[TimeRange] = None) -> bool: timerange: Optional[TimeRange] = None) -> bool:
""" """
Download trade history from the exchange. Download trade history from the exchange.
Appends to previously downloaded trades data. Appends to previously downloaded trades data.
@ -381,11 +382,11 @@ def download_trades_history(datadir: Path,
logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None') logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None')
logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None') logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None')
# Default since_ms to 30 days if nothing is given
new_trades = exchange.get_historic_trades(pair=pair, new_trades = exchange.get_historic_trades(pair=pair,
since=since if since else since=since if since else
int(arrow.utcnow().shift( int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000, days=-30).float_timestamp) * 1000,
# until=xxx,
from_id=from_id, from_id=from_id,
) )
trades.extend(new_trades[1]) trades.extend(new_trades[1])
@ -407,9 +408,9 @@ def download_trades_history(datadir: Path,
def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
timerange: TimeRange, erase=False) -> List[str]: timerange: TimeRange, erase=False) -> List[str]:
""" """
Refresh stored trades data. Refresh stored trades data for backtesting and hyperopt operations.
Used by freqtrade download-data Used by freqtrade download-data subcommand.
:return: Pairs not available :return: List of pairs that are not available.
""" """
pairs_not_available = [] pairs_not_available = []
for pair in pairs: for pair in pairs:
@ -425,9 +426,9 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
dl_file.unlink() dl_file.unlink()
logger.info(f'Downloading trades for pair {pair}.') logger.info(f'Downloading trades for pair {pair}.')
download_trades_history(datadir=datadir, exchange=exchange, _download_trades_history(datadir=datadir, exchange=exchange,
pair=pair, pair=pair,
timerange=timerange) timerange=timerange)
return pairs_not_available return pairs_not_available

2
freqtrade/utils.py
View File

@ -213,7 +213,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
else: else:
pairs_not_available = refresh_backtest_ohlcv_data( pairs_not_available = refresh_backtest_ohlcv_data(
exchange, pairs=config["pairs"], timeframes=config["timeframes"], exchange, pairs=config["pairs"], timeframes=config["timeframes"],
dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
except KeyboardInterrupt: except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...") sys.exit("SIGINT received, aborting ...")

56
tests/data/test_history.py
View File

@ -14,9 +14,9 @@ from freqtrade import OperationalException
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
from freqtrade.data import history from freqtrade.data import history
from freqtrade.data.history import (_load_cached_data_for_updating, from freqtrade.data.history import (_load_cached_data_for_updating,
_download_pair_history,
_download_trades_history,
convert_trades_to_ohlcv, convert_trades_to_ohlcv,
download_pair_history,
download_trades_history,
load_tickerdata_file, pair_data_filename, load_tickerdata_file, pair_data_filename,
pair_trades_filename, pair_trades_filename,
refresh_backtest_ohlcv_data, refresh_backtest_ohlcv_data,
@ -267,12 +267,12 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda
assert not file1_1.is_file() assert not file1_1.is_file()
assert not file2_1.is_file() assert not file2_1.is_file()
assert download_pair_history(datadir=testdatadir, exchange=exchange, assert _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='MEME/BTC', pair='MEME/BTC',
timeframe='1m') timeframe='1m')
assert download_pair_history(datadir=testdatadir, exchange=exchange, assert _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='CFI/BTC', pair='CFI/BTC',
timeframe='1m') timeframe='1m')
assert not exchange._pairs_last_refresh_time assert not exchange._pairs_last_refresh_time
assert file1_1.is_file() assert file1_1.is_file()
assert file2_1.is_file() assert file2_1.is_file()
@ -284,12 +284,12 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda
assert not file1_5.is_file() assert not file1_5.is_file()
assert not file2_5.is_file() assert not file2_5.is_file()
assert download_pair_history(datadir=testdatadir, exchange=exchange, assert _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='MEME/BTC', pair='MEME/BTC',
timeframe='5m') timeframe='5m')
assert download_pair_history(datadir=testdatadir, exchange=exchange, assert _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='CFI/BTC', pair='CFI/BTC',
timeframe='5m') timeframe='5m')
assert not exchange._pairs_last_refresh_time assert not exchange._pairs_last_refresh_time
assert file1_5.is_file() assert file1_5.is_file()
assert file2_5.is_file() assert file2_5.is_file()
@ -307,8 +307,8 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None)
mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick)
exchange = get_patched_exchange(mocker, default_conf) exchange = get_patched_exchange(mocker, default_conf)
download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m')
download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m') _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m')
assert json_dump_mock.call_count == 2 assert json_dump_mock.call_count == 2
@ -324,9 +324,9 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog,
_backup_file(file1_1) _backup_file(file1_1)
_backup_file(file1_5) _backup_file(file1_5)
assert not download_pair_history(datadir=testdatadir, exchange=exchange, assert not _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='MEME/BTC', pair='MEME/BTC',
timeframe='1m') timeframe='1m')
# clean files freshly downloaded # clean files freshly downloaded
_clean_test_file(file1_1) _clean_test_file(file1_1)
_clean_test_file(file1_5) _clean_test_file(file1_5)
@ -570,7 +570,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No
def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir): def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock())
mocker.patch( mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
) )
@ -580,7 +580,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test
ex = get_patched_exchange(mocker, default_conf) ex = get_patched_exchange(mocker, default_conf)
timerange = TimeRange.parse_timerange("20190101-20190102") timerange = TimeRange.parse_timerange("20190101-20190102")
refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
timeframes=["1m", "5m"], dl_path=testdatadir, timeframes=["1m", "5m"], datadir=testdatadir,
timerange=timerange, erase=True timerange=timerange, erase=True
) )
@ -591,7 +591,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test
def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock())
ex = get_patched_exchange(mocker, default_conf) ex = get_patched_exchange(mocker, default_conf)
mocker.patch( mocker.patch(
@ -600,7 +600,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
timerange = TimeRange.parse_timerange("20190101-20190102") timerange = TimeRange.parse_timerange("20190101-20190102")
unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["BTT/BTC", "LTC/USDT"], unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["BTT/BTC", "LTC/USDT"],
timeframes=["1m", "5m"], timeframes=["1m", "5m"],
dl_path=testdatadir, datadir=testdatadir,
timerange=timerange, erase=False timerange=timerange, erase=False
) )
@ -611,7 +611,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir): def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history.download_trades_history', MagicMock()) dl_mock = mocker.patch('freqtrade.data.history._download_trades_history', MagicMock())
mocker.patch( mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
) )
@ -646,8 +646,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
assert not file1.is_file() assert not file1.is_file()
assert download_trades_history(datadir=testdatadir, exchange=exchange, assert _download_trades_history(datadir=testdatadir, exchange=exchange,
pair='ETH/BTC') pair='ETH/BTC')
assert log_has("New Amount of trades: 5", caplog) assert log_has("New Amount of trades: 5", caplog)
assert file1.is_file() assert file1.is_file()
@ -657,8 +657,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
MagicMock(side_effect=ValueError)) MagicMock(side_effect=ValueError))
assert not download_trades_history(datadir=testdatadir, exchange=exchange, assert not _download_trades_history(datadir=testdatadir, exchange=exchange,
pair='ETH/BTC') pair='ETH/BTC')
assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)