adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

data/history cleanup

Browse Source
This commit is contained in:
hroff-1902 2019-05-17 19:05:36 +03:00
parent e0310906c7
commit e2b83624a3
2 changed files with 44 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
freqtrade/data/history.py
View File

@ -90,13 +90,8 @@ def load_pair_history(pair: str,
:return: DataFrame with ohlcv data :return: DataFrame with ohlcv data
""" """
# If the user force the refresh of pairs # The user forced the refresh of pairs
if refresh_pairs: if refresh_pairs:
if not exchange:
raise OperationalException("Exchange needs to be initialized when "
"calling load_data with refresh_pairs=True")
logger.info('Download data for pair and store them in %s', datadir)
download_pair_history(datadir=datadir, download_pair_history(datadir=datadir,
exchange=exchange, exchange=exchange,
pair=pair, pair=pair,
@ -115,10 +110,11 @@ def load_pair_history(pair: str,
arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S'))
return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing) return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing)
else: else:
logger.warning('No data for pair: "%s", Interval: %s. ' logger.warning(
'Use --refresh-pairs-cached option or download_backtest_data.py ' f'No history data for pair: "{pair}", interval: {ticker_interval}. '
'script to download the data', 'Use --refresh-pairs-cached option or download_backtest_data.py '
pair, ticker_interval) 'script to download the data'
)
return None return None
@ -190,7 +186,7 @@ def load_cached_data_for_updating(filename: Path, ticker_interval: str,
def download_pair_history(datadir: Optional[Path], def download_pair_history(datadir: Optional[Path],
exchange: Exchange, exchange: Optional[Exchange],
pair: str, pair: str,
ticker_interval: str = '5m', ticker_interval: str = '5m',
timerange: Optional[TimeRange] = None) -> bool: timerange: Optional[TimeRange] = None) -> bool:
@ -201,18 +197,26 @@ def download_pair_history(datadir: Optional[Path],
the full data will be redownloaded the full data will be redownloaded
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pair: pair to download :param pair: pair to download
:param ticker_interval: ticker interval :param ticker_interval: ticker interval
:param timerange: range of time to download :param timerange: range of time to download
:return: bool with success state :return: bool with success state
""" """
if not exchange:
raise OperationalException(
"Exchange needs to be initialized when downloading pair history data"
)
try: try:
path = make_testdata_path(datadir) path = make_testdata_path(datadir)
filepair = pair.replace("/", "_") filepair = pair.replace("/", "_")
filename = path.joinpath(f'{filepair}-{ticker_interval}.json') filename = path.joinpath(f'{filepair}-{ticker_interval}.json')
logger.info('Download the pair: "%s", Interval: %s', pair, ticker_interval) logger.info(
f'Download history data for pair: "{pair}", interval: {ticker_interval} '
f'and store in {datadir}.'
)
data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange) data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange)
@ -231,7 +235,9 @@ def download_pair_history(datadir: Optional[Path],
misc.file_dump_json(filename, data) misc.file_dump_json(filename, data)
return True return True
except BaseException:
logger.info('Failed to download the pair: "%s", Interval: %s', except Exception:
pair, ticker_interval) logger.error(
f'Failed to download history data for pair: "{pair}", interval: {ticker_interval}.'
)
return False return False

29
freqtrade/tests/data/test_history.py
View File

@ -59,7 +59,11 @@ def _clean_test_file(file: str) -> None:
def test_load_data_30min_ticker(mocker, caplog, default_conf) -> None: def test_load_data_30min_ticker(mocker, caplog, default_conf) -> None:
ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='30m', datadir=None) ld = history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='30m', datadir=None)
assert isinstance(ld, DataFrame) assert isinstance(ld, DataFrame)
assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 30m', caplog.record_tuples) assert not log_has(
'Download history data for pair: "UNITTEST/BTC", interval: 30m '
'and store in None.',
caplog.record_tuples
)
def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None: def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None:
@ -67,7 +71,7 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None:
assert not isinstance(ld, DataFrame) assert not isinstance(ld, DataFrame)
assert ld is None assert ld is None
assert log_has( assert log_has(
'No data for pair: "UNITTEST/BTC", Interval: 7m. ' 'No history data for pair: "UNITTEST/BTC", interval: 7m. '
'Use --refresh-pairs-cached option or download_backtest_data.py ' 'Use --refresh-pairs-cached option or download_backtest_data.py '
'script to download the data', 'script to download the data',
caplog.record_tuples caplog.record_tuples
@ -80,7 +84,11 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None:
_backup_file(file, copy_file=True) _backup_file(file, copy_file=True)
history.load_data(datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC']) history.load_data(datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC'])
assert os.path.isfile(file) is True assert os.path.isfile(file) is True
assert not log_has('Download the pair: "UNITTEST/BTC", Interval: 1m', caplog.record_tuples) assert not log_has(
'Download history data for pair: "UNITTEST/BTC", interval: 1m '
'and store in None.',
caplog.record_tuples
)
_clean_test_file(file) _clean_test_file(file)
@ -100,7 +108,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau
pair='MEME/BTC') pair='MEME/BTC')
assert os.path.isfile(file) is False assert os.path.isfile(file) is False
assert log_has( assert log_has(
'No data for pair: "MEME/BTC", Interval: 1m. ' 'No history data for pair: "MEME/BTC", interval: 1m. '
'Use --refresh-pairs-cached option or download_backtest_data.py ' 'Use --refresh-pairs-cached option or download_backtest_data.py '
'script to download the data', 'script to download the data',
caplog.record_tuples caplog.record_tuples
@ -113,7 +121,11 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau
exchange=exchange, exchange=exchange,
pair='MEME/BTC') pair='MEME/BTC')
assert os.path.isfile(file) is True assert os.path.isfile(file) is True
assert log_has('Download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) assert log_has(
'Download history data for pair: "MEME/BTC", interval: 1m '
'and store in None.',
caplog.record_tuples
)
with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'): with pytest.raises(OperationalException, match=r'Exchange needs to be initialized when.*'):
history.load_pair_history(datadir=None, history.load_pair_history(datadir=None,
ticker_interval='1m', ticker_interval='1m',
@ -293,7 +305,7 @@ def test_download_pair_history2(mocker, default_conf) -> None:
def test_download_backtesting_data_exception(ticker_history, mocker, caplog, default_conf) -> None: def test_download_backtesting_data_exception(ticker_history, mocker, caplog, default_conf) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_history', mocker.patch('freqtrade.exchange.Exchange.get_history',
side_effect=BaseException('File Error')) side_effect=Exception('File Error'))
exchange = get_patched_exchange(mocker, default_conf) exchange = get_patched_exchange(mocker, default_conf)
@ -308,7 +320,10 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, def
# clean files freshly downloaded # clean files freshly downloaded
_clean_test_file(file1_1) _clean_test_file(file1_1)
_clean_test_file(file1_5) _clean_test_file(file1_5)
assert log_has('Failed to download the pair: "MEME/BTC", Interval: 1m', caplog.record_tuples) assert log_has(
'Failed to download history data for pair: "MEME/BTC", interval: 1m.',
caplog.record_tuples
)
def test_load_tickerdata_file() -> None: def test_load_tickerdata_file() -> None: