adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2672 from hroff-1902/minor-data-history-2

Browse Source 
Minor: improvements in data/history.py
This commit is contained in:
Matthias 2019-12-17 09:22:56 +01:00 committed by GitHub
commit 1042f9847a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 62 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
freqtrade/data/history.py
View File

@ -155,11 +155,11 @@ def load_pair_history(pair: str,
# The user forced the refresh of pairs
if refresh_pairs:
download_pair_history(datadir=datadir,
exchange=exchange,
pair=pair,
timeframe=timeframe,
timerange=timerange)
_download_pair_history(datadir=datadir,
exchange=exchange,
pair=pair,
timeframe=timeframe,
timerange=timerange)
pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup)
@ -277,11 +277,11 @@ def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str,
return (data, since_ms)
def download_pair_history(datadir: Path,
exchange: Optional[Exchange],
pair: str,
timeframe: str = '5m',
timerange: Optional[TimeRange] = None) -> bool:
def _download_pair_history(datadir: Path,
exchange: Optional[Exchange],
pair: str,
timeframe: str = '5m',
timerange: Optional[TimeRange] = None) -> bool:
"""
Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that
@ -312,11 +312,12 @@ def download_pair_history(datadir: Path,
logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
# Default since_ms to 30 days if nothing is given
new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe,
since_ms=since_ms if since_ms
else
new_data = exchange.get_historic_ohlcv(pair=pair,
timeframe=timeframe,
since_ms=since_ms if since_ms else
int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000)
days=-30).float_timestamp) * 1000
)
data.extend(new_data)
logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))
@ -334,12 +335,12 @@ def download_pair_history(datadir: Path,
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
dl_path: Path, timerange: Optional[TimeRange] = None,
datadir: Path, timerange: Optional[TimeRange] = None,
erase=False) -> List[str]:
"""
Refresh stored ohlcv data for backtesting and hyperopt operations.
Used by freqtrade download-data
:return: Pairs not available
Used by freqtrade download-data subcommand.
:return: List of pairs that are not available.
"""
pairs_not_available = []
for pair in pairs:
@ -349,23 +350,23 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
continue
for timeframe in timeframes:
dl_file = pair_data_filename(dl_path, pair, timeframe)
dl_file = pair_data_filename(datadir, pair, timeframe)
if erase and dl_file.exists():
logger.info(
f'Deleting existing data for pair {pair}, interval {timeframe}.')
dl_file.unlink()
logger.info(f'Downloading pair {pair}, interval {timeframe}.')
download_pair_history(datadir=dl_path, exchange=exchange,
pair=pair, timeframe=str(timeframe),
timerange=timerange)
_download_pair_history(datadir=datadir, exchange=exchange,
pair=pair, timeframe=str(timeframe),
timerange=timerange)
return pairs_not_available
def download_trades_history(datadir: Path,
exchange: Exchange,
pair: str,
timerange: Optional[TimeRange] = None) -> bool:
def _download_trades_history(datadir: Path,
exchange: Exchange,
pair: str,
timerange: Optional[TimeRange] = None) -> bool:
"""
Download trade history from the exchange.
Appends to previously downloaded trades data.
@ -381,11 +382,11 @@ def download_trades_history(datadir: Path,
logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None')
logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None')
# Default since_ms to 30 days if nothing is given
new_trades = exchange.get_historic_trades(pair=pair,
since=since if since else
int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000,
# until=xxx,
from_id=from_id,
)
trades.extend(new_trades[1])
@ -407,9 +408,9 @@ def download_trades_history(datadir: Path,
def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
timerange: TimeRange, erase=False) -> List[str]:
"""
Refresh stored trades data.
Used by freqtrade download-data
:return: Pairs not available
Refresh stored trades data for backtesting and hyperopt operations.
Used by freqtrade download-data subcommand.
:return: List of pairs that are not available.
"""
pairs_not_available = []
for pair in pairs:
@ -425,9 +426,9 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
dl_file.unlink()
logger.info(f'Downloading trades for pair {pair}.')
download_trades_history(datadir=datadir, exchange=exchange,
pair=pair,
timerange=timerange)
_download_trades_history(datadir=datadir, exchange=exchange,
pair=pair,
timerange=timerange)
return pairs_not_available

2
freqtrade/utils.py
View File

@ -213,7 +213,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
else:
pairs_not_available = refresh_backtest_ohlcv_data(
exchange, pairs=config["pairs"], timeframes=config["timeframes"],
dl_path=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase"))
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")

56
tests/data/test_history.py
View File

@ -14,9 +14,9 @@ from freqtrade import OperationalException
from freqtrade.configuration import TimeRange
from freqtrade.data import history
from freqtrade.data.history import (_load_cached_data_for_updating,
_download_pair_history,
_download_trades_history,
convert_trades_to_ohlcv,
download_pair_history,
download_trades_history,
load_tickerdata_file, pair_data_filename,
pair_trades_filename,
refresh_backtest_ohlcv_data,
@ -267,12 +267,12 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda
assert not file1_1.is_file()
assert not file2_1.is_file()
assert download_pair_history(datadir=testdatadir, exchange=exchange,
pair='MEME/BTC',
timeframe='1m')
assert download_pair_history(datadir=testdatadir, exchange=exchange,
pair='CFI/BTC',
timeframe='1m')
assert _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='MEME/BTC',
timeframe='1m')
assert _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='CFI/BTC',
timeframe='1m')
assert not exchange._pairs_last_refresh_time
assert file1_1.is_file()
assert file2_1.is_file()
@ -284,12 +284,12 @@ def test_download_pair_history(ticker_history_list, mocker, default_conf, testda
assert not file1_5.is_file()
assert not file2_5.is_file()
assert download_pair_history(datadir=testdatadir, exchange=exchange,
pair='MEME/BTC',
timeframe='5m')
assert download_pair_history(datadir=testdatadir, exchange=exchange,
pair='CFI/BTC',
timeframe='5m')
assert _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='MEME/BTC',
timeframe='5m')
assert _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='CFI/BTC',
timeframe='5m')
assert not exchange._pairs_last_refresh_time
assert file1_5.is_file()
assert file2_5.is_file()
@ -307,8 +307,8 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None:
json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None)
mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick)
exchange = get_patched_exchange(mocker, default_conf)
download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m')
download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m')
_download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m')
_download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='3m')
assert json_dump_mock.call_count == 2
@ -324,9 +324,9 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog,
_backup_file(file1_1)
_backup_file(file1_5)
assert not download_pair_history(datadir=testdatadir, exchange=exchange,
pair='MEME/BTC',
timeframe='1m')
assert not _download_pair_history(datadir=testdatadir, exchange=exchange,
pair='MEME/BTC',
timeframe='1m')
# clean files freshly downloaded
_clean_test_file(file1_1)
_clean_test_file(file1_5)
@ -570,7 +570,7 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No
def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock())
dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock())
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
)
@ -580,7 +580,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test
ex = get_patched_exchange(mocker, default_conf)
timerange = TimeRange.parse_timerange("20190101-20190102")
refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"],
timeframes=["1m", "5m"], dl_path=testdatadir,
timeframes=["1m", "5m"], datadir=testdatadir,
timerange=timerange, erase=True
)
@ -591,7 +591,7 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test
def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock())
dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock())
ex = get_patched_exchange(mocker, default_conf)
mocker.patch(
@ -600,7 +600,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
timerange = TimeRange.parse_timerange("20190101-20190102")
unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["BTT/BTC", "LTC/USDT"],
timeframes=["1m", "5m"],
dl_path=testdatadir,
datadir=testdatadir,
timerange=timerange, erase=False
)
@ -611,7 +611,7 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir):
def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir):
dl_mock = mocker.patch('freqtrade.data.history.download_trades_history', MagicMock())
dl_mock = mocker.patch('freqtrade.data.history._download_trades_history', MagicMock())
mocker.patch(
'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)
)
@ -646,8 +646,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
assert not file1.is_file()
assert download_trades_history(datadir=testdatadir, exchange=exchange,
pair='ETH/BTC')
assert _download_trades_history(datadir=testdatadir, exchange=exchange,
pair='ETH/BTC')
assert log_has("New Amount of trades: 5", caplog)
assert file1.is_file()
@ -657,8 +657,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
mocker.patch('freqtrade.exchange.Exchange.get_historic_trades',
MagicMock(side_effect=ValueError))
assert not download_trades_history(datadir=testdatadir, exchange=exchange,
pair='ETH/BTC')
assert not _download_trades_history(datadir=testdatadir, exchange=exchange,
pair='ETH/BTC')
assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)