adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into feat/externalsignals

Browse Source
This commit is contained in:
Timothy Pogue
2022-09-05 15:10:25 -06:00
parent 8f261d8edf 9fb3517adc
commit cd8455ccb7
106 changed files with 3575 additions and 1692 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
freqtrade/data/dataprovider.py
View File

@@ -210,9 +210,9 @@ class DataProvider:
timerange = TimeRange.parse_timerange(None if self._config.get(
'timerange') is None else str(self._config.get('timerange')))
# Move informative start time respecting startup_candle_count
timerange.subtract_start(
timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0)
)
startup_candles = self.get_required_startup(str(timeframe))
tf_seconds = timeframe_to_seconds(str(timeframe))
timerange.subtract_start(tf_seconds * startup_candles)
self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
pair=pair,
timeframe=timeframe or self._config['timeframe'],
@@ -224,6 +224,21 @@ class DataProvider:
)
return self.__cached_pairs_backtesting[saved_pair].copy()
def get_required_startup(self, timeframe: str) -> int:
freqai_config = self._config.get('freqai', {})
if not freqai_config.get('enabled', False):
return self._config.get('startup_candle_count', 0)
else:
startup_candles = self._config.get('startup_candle_count', 0)
indicator_periods = freqai_config['feature_parameters']['indicator_periods_candles']
# make sure the startupcandles is at least the set maximum indicator periods
self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods))
tf_seconds = timeframe_to_seconds(timeframe)
train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds
total_candles = int(self._config['startup_candle_count'] + train_candles)
logger.info(f'Increasing startup_candle_count for freqai to {total_candles}')
return total_candles
def get_pair_dataframe(
self,
pair: str,

28
freqtrade/data/history/hdf5datahandler.py
View File

@@ -7,9 +7,8 @@ import numpy as np
import pandas as pd
from freqtrade.configuration import TimeRange
from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS,
ListPairsWithTimeframes, TradeList)
from freqtrade.enums import CandleType, TradingMode
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
from freqtrade.enums import CandleType
from .idatahandler import IDataHandler
@@ -21,29 +20,6 @@ class HDF5DataHandler(IDataHandler):
_columns = DEFAULT_DATAFRAME_COLUMNS
@classmethod
def ohlcv_get_available_data(
cls, datadir: Path, trading_mode: TradingMode) -> ListPairsWithTimeframes:
"""
Returns a list of all pairs with ohlcv data available in this datadir
:param datadir: Directory to search for ohlcv files
:param trading_mode: trading-mode to be used
:return: List of Tuples of (pair, timeframe)
"""
if trading_mode == TradingMode.FUTURES:
datadir = datadir.joinpath('futures')
_tmp = [
re.search(
cls._OHLCV_REGEX, p.name
) for p in datadir.glob("*.h5")
]
return [
(
cls.rebuild_pair_from_filename(match[1]),
cls.rebuild_timeframe_from_filename(match[2]),
CandleType.from_string(match[3])
) for match in _tmp if match and len(match.groups()) > 1]
@classmethod
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]:
"""

10
freqtrade/data/history/history_utils.py
View File

@@ -302,8 +302,8 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
if trading_mode == 'futures':
# Predefined candletype (and timeframe) depending on exchange
# Downloads what is necessary to backtest based on futures data.
tf_mark = exchange._ft_has['mark_ohlcv_timeframe']
fr_candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price'])
tf_mark = exchange.get_option('mark_ohlcv_timeframe')
fr_candle_type = CandleType.from_string(exchange.get_option('mark_ohlcv_price'))
# All exchanges need FundingRate for futures trading.
# The timeframe is aligned to the mark-price timeframe.
for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type):
@@ -330,13 +330,12 @@ def _download_trades_history(exchange: Exchange,
try:
until = None
since = 0
if timerange:
if timerange.starttype == 'date':
since = timerange.startts * 1000
if timerange.stoptype == 'date':
until = timerange.stopts * 1000
else:
since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000
trades = data_handler.trades_load(pair)
@@ -349,6 +348,9 @@ def _download_trades_history(exchange: Exchange,
logger.info(f"Start earlier than available data. Redownloading trades for {pair}...")
trades = []
if not since:
since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000
from_id = trades[-1][1] if trades else None
if trades and since < trades[-1][0]:
# Reset since to the last available point

29
freqtrade/data/history/idatahandler.py
View File

@@ -9,7 +9,7 @@ from abc import ABC, abstractmethod
from copy import deepcopy
from datetime import datetime, timezone
from pathlib import Path
from typing import List, Optional, Type
from typing import List, Optional, Tuple, Type
from pandas import DataFrame
@@ -39,15 +39,26 @@ class IDataHandler(ABC):
raise NotImplementedError()
@classmethod
@abstractmethod
def ohlcv_get_available_data(
cls, datadir: Path, trading_mode: TradingMode) -> ListPairsWithTimeframes:
"""
Returns a list of all pairs with ohlcv data available in this datadir
:param datadir: Directory to search for ohlcv files
:param trading_mode: trading-mode to be used
:return: List of Tuples of (pair, timeframe)
:return: List of Tuples of (pair, timeframe, CandleType)
"""
if trading_mode == TradingMode.FUTURES:
datadir = datadir.joinpath('futures')
_tmp = [
re.search(
cls._OHLCV_REGEX, p.name
) for p in datadir.glob(f"*.{cls._get_file_extension()}")]
return [
(
cls.rebuild_pair_from_filename(match[1]),
cls.rebuild_timeframe_from_filename(match[2]),
CandleType.from_string(match[3])
) for match in _tmp if match and len(match.groups()) > 1]
@classmethod
@abstractmethod
@@ -73,6 +84,18 @@ class IDataHandler(ABC):
:return: None
"""
def ohlcv_data_min_max(self, pair: str, timeframe: str,
candle_type: CandleType) -> Tuple[datetime, datetime]:
"""
Returns the min and max timestamp for the given pair and timeframe.
:param pair: Pair to get min/max for
:param timeframe: Timeframe to get min/max for
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: (min, max)
"""
data = self._ohlcv_load(pair, timeframe, None, candle_type)
return data.iloc[0]['date'].to_pydatetime(), data.iloc[-1]['date'].to_pydatetime()
@abstractmethod
def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange],
candle_type: CandleType

26
freqtrade/data/history/jsondatahandler.py
View File

@@ -8,9 +8,9 @@ from pandas import DataFrame, read_json, to_datetime
from freqtrade import misc
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, ListPairsWithTimeframes, TradeList
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, TradeList
from freqtrade.data.converter import trades_dict_to_list
from freqtrade.enums import CandleType, TradingMode
from freqtrade.enums import CandleType
from .idatahandler import IDataHandler
@@ -23,28 +23,6 @@ class JsonDataHandler(IDataHandler):
_use_zip = False
_columns = DEFAULT_DATAFRAME_COLUMNS
@classmethod
def ohlcv_get_available_data(
cls, datadir: Path, trading_mode: TradingMode) -> ListPairsWithTimeframes:
"""
Returns a list of all pairs with ohlcv data available in this datadir
:param datadir: Directory to search for ohlcv files
:param trading_mode: trading-mode to be used
:return: List of Tuples of (pair, timeframe)
"""
if trading_mode == 'futures':
datadir = datadir.joinpath('futures')
_tmp = [
re.search(
cls._OHLCV_REGEX, p.name
) for p in datadir.glob(f"*.{cls._get_file_extension()}")]
return [
(
cls.rebuild_pair_from_filename(match[1]),
cls.rebuild_timeframe_from_filename(match[2]),
CandleType.from_string(match[3])
) for match in _tmp if match and len(match.groups()) > 1]
@classmethod
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]:
"""