adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rewrite _download_pair_history to support an endtime.

Browse Source
This commit is contained in:
Kuy Krawczeniuk 2021-03-25 16:10:32 -07:00
parent d52d9be59b
commit f7eba77e36
1 changed files with 86 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
freqtrade/data/history/history_utils.py
View File

@ -152,71 +152,114 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
return data, start_ms return data, start_ms
def _download_pair_history(datadir: Path, def _download_pair_history(datadir: Path, exchange: Exchange, pair: str, *,
exchange: Exchange,
pair: str, *,
timeframe: str = '5m', timeframe: str = '5m',
timerange: Optional[TimeRange] = None, since: datetime = None, until: datetime = None,
data_handler: IDataHandler = None) -> bool: data_handler: IDataHandler = None,
""" fill_gaps: bool = False) -> bool:
Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that
exists in a cache. If timerange starts earlier than the data in the cache,
the full data will be redownloaded
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pair: pair to download
:param timeframe: Timeframe (e.g "5m")
:param timerange: range of time to download
:return: bool with success state
"""
data_handler = get_datahandler(datadir, data_handler=data_handler) data_handler = get_datahandler(datadir, data_handler=data_handler)
try: try:
logger.info( logger.info(
f'Download history data for pair: "{pair}", timeframe: {timeframe} ' f'Downloading history data for par: "{pair}", timeframe: '
f'and store in {datadir}.' f'{timeframe} and storing in "{datadir}"'
) )
# data, since_ms = _load_cached_data_for_updating_old(datadir, pair, timeframe, timerange) # Load existing data for updating
data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange, cached = data_handler.ohlcv_load(pair, timeframe=timeframe,
data_handler=data_handler) timerange=None, fill_missing=False,
warn_no_data=False)
logger.debug("Current Start: %s", cached_start, cached_end = None, None
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') if not cached.empty:
logger.debug("Current End: %s", cached_start = cached.iloc[0]['date']
f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') cached_end = cached.iloc[-1]['date']
logger.debug("Cached Start: %s",
f"{cached_start:%Y-%m-%d %H:%M:%S}" if cached_start else 'None')
logger.debug("Cached End: %s",
f"{cached_end:%Y-%m-%d %H:%M:%S}" if not cached_end else 'None')
since_ms, until_ms = None, None
if cached.empty:
if since:
since_ms = since.timestamp() * 1000
else:
since_ms = arrow.utcnow().shift(days=-30).float_timestamp * \
1000
if until:
until_ms = until.timestamp() * 1000
else:
until_ms = datetime.now(timezone.utc).timestamp() * 1000
else:
if since:
if since < cached_start:
since_ms = since.timestamp() * 1000
elif cached_start <= since < cached_end:
logger.warning("The timerange overlaps with cached data."
" This may lead to unexpected outcomes "
"including overwriting existing data!")
since_ms = since.timestamp() * 1000
else:
if fill_gaps:
since_ms = cached_end.timestamp() * 1000
else:
logger.warning("The timerange starts after cached "
"data. There will be a gap in the "
"saved data! Use --fill-gaps to "
"prevent this!")
since_ms = since.timestamp() * 1000
else:
since_ms = cached_end.timestamp() * 1000
if until:
if until < cached_start:
if fill_gaps:
until_ms = cached_start.timestamp() * 1000
else:
logger.warning("The timerange ends before the cached "
"data. There will be a gap in the "
"saved data! Use --fill-gaps to "
"prevent this!")
until_ms = until.timestamp() * 1000
elif cached_start < until <= cached_end:
logger.warning("The timerange overlaps with cached data."
" This may lead to unexpected outcomes "
"including overwriting existing data!")
until_ms = until.timestamp() * 1000
else:
until_ms = until.timestamp()
else:
until_ms = datetime.now(timezone.utc).timestamp() * 1000
# Default since_ms to 30 days if nothing is given
new_data = exchange.get_historic_ohlcv(pair=pair, new_data = exchange.get_historic_ohlcv(pair=pair,
timeframe=timeframe, timeframe=timeframe,
since_ms=since_ms if since_ms else since_ms=since_ms,
int(arrow.utcnow().shift( until_ms=until_ms)
days=-30).float_timestamp) * 1000
) new_df = ohlcv_to_dataframe(new_data, timeframe, pair,
# TODO: Maybe move parsing to exchange class (?) fill_missing=False)
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
fill_missing=False, drop_incomplete=True) if cached.empty:
if data.empty: data = new_df
data = new_dataframe
else: else:
# Run cleaning again to ensure there were no duplicate candles data = clean_ohlcv_dataframe(cached.append(new_df), timeframe,
# Especially between existing and new data. pair, fill_missing=True,
data = clean_ohlcv_dataframe(data.append(new_dataframe), timeframe, pair, drop_incomplete=False)
fill_missing=False, drop_incomplete=False)
logger.debug("New Start: %s", logger.debug("New Start: %s",
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
logger.debug("New End: %s", logger.debug("New End: %s",
f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
data_handler.ohlcv_store(pair, timeframe, data=data) data_handler.ohlcv_store(pair, timeframe, data)
return True return True
except Exception: except Exception:
logger.exception( logger.exception(
f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}.' f'Failed to download history data for pari: "{pair}", '
f'timeframe: {timeframe}.'
) )
return False return False