adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2672 from hroff-1902/minor-data-history-2

Browse Source 
Minor: improvements in data/history.py
This commit is contained in:
Matthias 2019-12-17 09:22:56 +01:00 committed by GitHub
commit 1042f9847a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 62 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
freqtrade/data/history.py
View File

@ -155,7 +155,7 @@ def load_pair_history(pair: str,
# The user forced the refresh of pairs # The user forced the refresh of pairs
if refresh_pairs: if refresh_pairs:
download_pair_history(datadir=datadir, _download_pair_history(datadir=datadir,
exchange=exchange, exchange=exchange,
pair=pair, pair=pair,
timeframe=timeframe, timeframe=timeframe,
@ -277,7 +277,7 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str,
return (data, since_ms) return (data, since_ms)
def download_pair_history(datadir: Path, def _download_pair_history(datadir: Path,
exchange: Optional[Exchange], exchange: Optional[Exchange],
pair: str, pair: str,
timeframe: str = '5m', timeframe: str = '5m',
@ -312,11 +312,12 @@ def download_pair_history(datadir: Path,
logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
# Default since_ms to 30 days if nothing is given # Default since_ms to 30 days if nothing is given
new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, new_data = exchange.get_historic_ohlcv(pair=pair,
since_ms=since_ms if since_ms timeframe=timeframe,
else since_ms=since_ms if since_ms else
int(arrow.utcnow().shift( int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000) days=-30).float_timestamp) * 1000
)
data.extend(new_data) data.extend(new_data)
logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))
@ -334,12 +335,12 @@ def download_pair_history(datadir: Path,
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
dl_path: Path, timerange: Optional[TimeRange] = None, datadir: Path, timerange: Optional[TimeRange] = None,
erase=False) -> List[str]: erase=False) -> List[str]:
""" """
Refresh stored ohlcv data for backtesting and hyperopt operations. Refresh stored ohlcv data for backtesting and hyperopt operations.
Used by freqtrade download-data Used by freqtrade download-data subcommand.
:return: Pairs not available :return: List of pairs that are not available.
""" """
pairs_not_available = [] pairs_not_available = []
for pair in pairs: for pair in pairs:
@ -349,20 +350,20 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
continue continue
for timeframe in timeframes: for timeframe in timeframes:
dl_file = pair_data_filename(dl_path, pair, timeframe) dl_file = pair_data_filename(datadir, pair, timeframe)
if erase and dl_file.exists(): if erase and dl_file.exists():
logger.info( logger.info(
f'Deleting existing data for pair {pair}, interval {timeframe}.') f'Deleting existing data for pair {pair}, interval {timeframe}.')
dl_file.unlink() dl_file.unlink()
logger.info(f'Downloading pair {pair}, interval {timeframe}.') logger.info(f'Downloading pair {pair}, interval {timeframe}.')
download_pair_history(datadir=dl_path, exchange=exchange, _download_pair_history(datadir=datadir, exchange=exchange,
pair=pair, timeframe=str(timeframe), pair=pair, timeframe=str(timeframe),
timerange=timerange) timerange=timerange)
return pairs_not_available return pairs_not_available
def download_trades_history(datadir: Path, def _download_trades_history(datadir: Path,
exchange: Exchange, exchange: Exchange,
pair: str, pair: str,
timerange: Optional[TimeRange] = None) -> bool: timerange: Optional[TimeRange] = None) -> bool:
@ -381,11 +382,11 @@ def download_trades_history(datadir: Path,
logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None') logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None')
logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None') logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None')
# Default since_ms to 30 days if nothing is given
new_trades = exchange.get_historic_trades(pair=pair, new_trades = exchange.get_historic_trades(pair=pair,
since=since if since else since=since if since else
int(arrow.utcnow().shift( int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000, days=-30).float_timestamp) * 1000,
# until=xxx,
from_id=from_id, from_id=from_id,
) )
trades.extend(new_trades[1]) trades.extend(new_trades[1])
@ -407,9 +408,9 @@ def download_trades_history(datadir: Path,
def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
timerange: TimeRange, erase=False) -> List[str]: timerange: TimeRange, erase=False) -> List[str]:
""" """
Refresh stored trades data. Refresh stored trades data for backtesting and hyperopt operations.
Used by freqtrade download-data Used by freqtrade download-data subcommand.
:return: Pairs not available :return: List of pairs that are not available.
""" """
pairs_not_available = [] pairs_not_available = []
for pair in pairs: for pair in pairs:
@ -425,7 +426,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
dl_file.unlink() dl_file.unlink()
logger.info(f'Downloading trades for pair {pair}.') logger.info(f'Downloading trades for pair {pair}.')
download_trades_history(datadir=datadir, exchange=exchange, _download_trades_history(datadir=datadir, exchange=exchange,
pair=pair, pair=pair,
timerange=timerange) timerange=timerange)
return pairs_not_available return pairs_not_available

2
freqtrade/utils.py
View File

@ -213,7 +213,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
else: else:
pairs_not_available = refresh_backtest_ohlcv_data( pairs_not_available = refresh_backtest_ohlcv_data(
exchange, pairs=config["pairs"], timeframes=config["timeframes"], exchange, pairs=config["pairs"], timeframes=config["timeframes"],
dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
except KeyboardInterrupt: except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...") sys.exit("SIGINT received, aborting ...")

32
tests/data/test_history.py
View File

@ -14,9 +14,9 @@ from freqtrade import OperationalException
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
from freqtrade.data import history from freqtrade.data import history
from freqtrade.data.history import (_load_cached_data_for_updating, from freqtrade.data.history import (_load_cached_data_for_updating,
_download_pair_history,
_download_trades_history,
convert_trades_to_ohlcv, convert_trades_to_ohlcv,
download_pair_history,
download_trades_history,
load_tickerdata_file, pair_data_filename, load_tickerdata_file, pair_data_filename,
pair_trades_filename, pair_trades_filename,
refresh_backtest_ohlcv_data, refresh_backtest_ohlcv_data,
@ -267,10 +267,10 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda
assert not file1_1.is_file() assert not file1_1.is_file()
assert not file2_1.is_file() assert not file2_1.is_file()
assert download_pair_history(datadir=testdatadir, exchange=exchange, assert _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='MEME/BTC', pair='MEME/BTC',
timeframe='1m') timeframe='1m')
assert download_pair_history(datadir=testdatadir, exchange=exchange, assert _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='CFI/BTC', pair='CFI/BTC',
timeframe='1m') timeframe='1m')
assert not exchange._pairs_last_refresh_time assert not exchange._pairs_last_refresh_time
@ -284,10 +284,10 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda
assert not file1_5.is_file() assert not file1_5.is_file()
assert not file2_5.is_file() assert not file2_5.is_file()
assert download_pair_history(datadir=testdatadir, exchange=exchange, assert _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='MEME/BTC', pair='MEME/BTC',
timeframe='5m') timeframe='5m')
assert download_pair_history(datadir=testdatadir, exchange=exchange, assert _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='CFI/BTC', pair='CFI/BTC',
timeframe='5m') timeframe='5m')
assert not exchange._pairs_last_refresh_time assert not exchange._pairs_last_refresh_time
@ -307,8 +307,8 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None)
mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick)
exchange = get_patched_exchange(mocker, default_conf) exchange = get_patched_exchange(mocker, default_conf)
download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m')
download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m') _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m')
assert json_dump_mock.call_count == 2 assert json_dump_mock.call_count == 2
@ -324,7 +324,7 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog,
_backup_file(file1_1) _backup_file(file1_1)
_backup_file(file1_5) _backup_file(file1_5)
assert not download_pair_history(datadir=testdatadir, exchange=exchange, assert not _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='MEME/BTC', pair='MEME/BTC',
timeframe='1m') timeframe='1m')
# clean files freshly downloaded # clean files freshly downloaded
@ -570,7 +570,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No
def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir): def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock())
mocker.patch( mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
) )
@ -580,7 +580,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test
ex = get_patched_exchange(mocker, default_conf) ex = get_patched_exchange(mocker, default_conf)
timerange = TimeRange.parse_timerange("20190101-20190102") timerange = TimeRange.parse_timerange("20190101-20190102")
refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
timeframes=["1m", "5m"], dl_path=testdatadir, timeframes=["1m", "5m"], datadir=testdatadir,
timerange=timerange, erase=True timerange=timerange, erase=True
) )
@ -591,7 +591,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test
def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock())
ex = get_patched_exchange(mocker, default_conf) ex = get_patched_exchange(mocker, default_conf)
mocker.patch( mocker.patch(
@ -600,7 +600,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
timerange = TimeRange.parse_timerange("20190101-20190102") timerange = TimeRange.parse_timerange("20190101-20190102")
unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["BTT/BTC", "LTC/USDT"], unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["BTT/BTC", "LTC/USDT"],
timeframes=["1m", "5m"], timeframes=["1m", "5m"],
dl_path=testdatadir, datadir=testdatadir,
timerange=timerange, erase=False timerange=timerange, erase=False
) )
@ -611,7 +611,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir): def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history.download_trades_history', MagicMock()) dl_mock = mocker.patch('freqtrade.data.history._download_trades_history', MagicMock())
mocker.patch( mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
) )
@ -646,7 +646,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
assert not file1.is_file() assert not file1.is_file()
assert download_trades_history(datadir=testdatadir, exchange=exchange, assert _download_trades_history(datadir=testdatadir, exchange=exchange,
pair='ETH/BTC') pair='ETH/BTC')
assert log_has("New Amount of trades: 5", caplog) assert log_has("New Amount of trades: 5", caplog)
assert file1.is_file() assert file1.is_file()
@ -657,7 +657,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
MagicMock(side_effect=ValueError)) MagicMock(side_effect=ValueError))
assert not download_trades_history(datadir=testdatadir, exchange=exchange, assert not _download_trades_history(datadir=testdatadir, exchange=exchange,
pair='ETH/BTC') pair='ETH/BTC')
assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)