adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

data/history cleanup

Browse Source
This commit is contained in:
hroff-1902 2019-05-17 19:05:36 +03:00
parent e0310906c7
commit e2b83624a3
2 changed files with 44 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
freqtrade/data/history.py
View File

@ -90,13 +90,8 @@ def load_pair_history(pair: str,
:return: DataFrame with ohlcv data
"""
# If the user force the refresh of pairs
# The user forced the refresh of pairs
if refresh_pairs:
if not exchange:
raise OperationalException("Exchange needs to be initialized when "
"calling load_data with refresh_pairs=True")
logger.info('Download data for pair and store them in %s', datadir)
download_pair_history(datadir=datadir,
exchange=exchange,
pair=pair,
@ -115,10 +110,11 @@ def load_pair_history(pair: str,
arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S'))
return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing)
else:
logger.warning('No data for pair: "%s", Interval: %s. '
'Use --refresh-pairs-cached option or download_backtest_data.py '
'script to download the data',
pair, ticker_interval)
logger.warning(
f'No history data for pair: "{pair}", interval: {ticker_interval}. '
'Use --refresh-pairs-cached option or download_backtest_data.py '
'script to download the data'
)
return None
@ -190,7 +186,7 @@ def load_cached_data_for_updating(filename: Path, ticker_interval: str,
def download_pair_history(datadir: Optional[Path],
exchange: Exchange,
exchange: Optional[Exchange],
pair: str,
ticker_interval: str = '5m',
timerange: Optional[TimeRange] = None) -> bool:
@ -201,18 +197,26 @@ def download_pair_history(datadir: Optional[Path],
the full data will be redownloaded
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pair: pair to download
:param ticker_interval: ticker interval
:param timerange: range of time to download
:return: bool with success state
"""
if not exchange:
raise OperationalException(
"Exchange needs to be initialized when downloading pair history data"
)
try:
path = make_testdata_path(datadir)
filepair = pair.replace("/", "_")
filename = path.joinpath(f'{filepair}-{ticker_interval}.json')
logger.info('Download the pair: "%s", Interval: %s', pair, ticker_interval)
logger.info(
f'Download history data for pair: "{pair}", interval: {ticker_interval} '
f'and store in {datadir}.'
)
data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange)
@ -231,7 +235,9 @@ def download_pair_history(datadir: Optional[Path],
misc.file_dump_json(filename, data)
return True
except BaseException:
logger.info('Failed to download the pair: "%s", Interval: %s',
pair, ticker_interval)
except Exception:
logger.error(
f'Failed to download history data for pair: "{pair}", interval: {ticker_interval}.'
)
return False

29
freqtrade/tests/data/test_history.py
View File

@ -59,7 +59,11 @@ def _clean_test_file(file: str) -> None:
def test_load_data_30min_ticker(mocker, caplog, default_conf) -> None:
ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='30m', datadir=None)
assert isinstance(ld, DataFrame)
assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 30m', caplog.record_tuples)
assert not log_has(
'Download history data for pair: "UNITTEST/BTC", interval: 30m '
'and store in None.',
caplog.record_tuples
)
def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None:
@ -67,7 +71,7 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None:
assert not isinstance(ld, DataFrame)
assert ld is None
assert log_has(
'No data for pair: "UNITTEST/BTC", Interval: 7m. '
'No history data for pair: "UNITTEST/BTC", interval: 7m. '
'Use --refresh-pairs-cached option or download_backtest_data.py '
'script to download the data',
caplog.record_tuples
@ -80,7 +84,11 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None:
_backup_file(file, copy_file=True)
history.load_data(datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC'])
assert os.path.isfile(file) is True
assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 1m', caplog.record_tuples)
assert not log_has(
'Download history data for pair: "UNITTEST/BTC", interval: 1m '
'and store in None.',
caplog.record_tuples
)
_clean_test_file(file)
@ -100,7 +108,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau
pair='MEME/BTC')
assert os.path.isfile(file) is False
assert log_has(
'No data for pair: "MEME/BTC", Interval: 1m. '
'No history data for pair: "MEME/BTC", interval: 1m. '
'Use --refresh-pairs-cached option or download_backtest_data.py '
'script to download the data',
caplog.record_tuples
@ -113,7 +121,11 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau
exchange=exchange,
pair='MEME/BTC')
assert os.path.isfile(file) is True
assert log_has('Download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples)
assert log_has(
'Download history data for pair: "MEME/BTC", interval: 1m '
'and store in None.',
caplog.record_tuples
)
with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'):
history.load_pair_history(datadir=None,
ticker_interval='1m',
@ -293,7 +305,7 @@ def test_download_pair_history2(mocker, default_conf) -> None:
def test_download_backtesting_data_exception(ticker_history, mocker, caplog, default_conf) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_history',
side_effect=BaseException('File Error'))
side_effect=Exception('File Error'))
exchange = get_patched_exchange(mocker, default_conf)
@ -308,7 +320,10 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, def
# clean files freshly downloaded
_clean_test_file(file1_1)
_clean_test_file(file1_5)
assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples)
assert log_has(
'Failed to download history data for pair: "MEME/BTC", interval: 1m.',
caplog.record_tuples
)
def test_load_tickerdata_file() -> None: