adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4630 from freqtrade/reduce_dataload_verbosity

Browse Source 
Reduce verbosity of missing data if less than 1% of data is missing
This commit is contained in:
Matthias 2021-03-30 16:18:01 +02:00 committed by GitHub
commit 89cfcc8ba6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
freqtrade/data/converter.py
View File

@ -110,8 +110,15 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
df.reset_index(inplace=True) df.reset_index(inplace=True)
len_before = len(dataframe) len_before = len(dataframe)
len_after = len(df) len_after = len(df)
pct_missing = (len_after - len_before) / len_before if len_before > 0 else 0
if len_before != len_after: if len_before != len_after:
logger.info(f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}") message = (f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}"
f" - {round(pct_missing * 100, 2)} %")
if pct_missing > 0.01:
logger.info(message)
else:
# Don't be verbose if only a small amount is missing
logger.debug(message)
return df return df

10
tests/data/test_converter.py
View File

@ -10,7 +10,7 @@ from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_forma
trades_to_ohlcv, trim_dataframe) trades_to_ohlcv, trim_dataframe)
from freqtrade.data.history import (get_timerange, load_data, load_pair_history, from freqtrade.data.history import (get_timerange, load_data, load_pair_history,
validate_backtest_data) validate_backtest_data)
from tests.conftest import log_has from tests.conftest import log_has, log_has_re
from tests.data.test_history import _backup_file, _clean_test_file from tests.data.test_history import _backup_file, _clean_test_file
@ -62,8 +62,8 @@ def test_ohlcv_fill_up_missing_data(testdatadir, caplog):
# Column names should not change # Column names should not change
assert (data.columns == data2.columns).all() assert (data.columns == data2.columns).all()
assert log_has(f"Missing data fillup for UNITTEST/BTC: before: " assert log_has_re(f"Missing data fillup for UNITTEST/BTC: before: "
f"{len(data)} - after: {len(data2)}", caplog) f"{len(data)} - after: {len(data2)}.*", caplog)
# Test fillup actually fixes invalid backtest data # Test fillup actually fixes invalid backtest data
min_date, max_date = get_timerange({'UNITTEST/BTC': data}) min_date, max_date = get_timerange({'UNITTEST/BTC': data})
@ -125,8 +125,8 @@ def test_ohlcv_fill_up_missing_data2(caplog):
# Column names should not change # Column names should not change
assert (data.columns == data2.columns).all() assert (data.columns == data2.columns).all()
assert log_has(f"Missing data fillup for UNITTEST/BTC: before: " assert log_has_re(f"Missing data fillup for UNITTEST/BTC: before: "
f"{len(data)} - after: {len(data2)}", caplog) f"{len(data)} - after: {len(data2)}.*", caplog)
def test_ohlcv_drop_incomplete(caplog): def test_ohlcv_drop_incomplete(caplog):