adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2430 from freqtrade/startup_period_bt

Browse Source 
Add Startup period for strategies
This commit is contained in:
hroff-1902
2019-10-28 23:33:30 +03:00
committed by GitHub
parent 907baea8b2 61c037f2cf
commit 5254059fe4
18 changed files with 321 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
freqtrade/configuration/timerange.py
View File

@@ -1,11 +1,14 @@
"""
This module contains the argument manager class
"""
import logging
import re
from typing import Optional
import arrow
logger = logging.getLogger(__name__)
class TimeRange:
"""
@@ -27,6 +30,34 @@ class TimeRange:
return (self.starttype == other.starttype and self.stoptype == other.stoptype
and self.startts == other.startts and self.stopts == other.stopts)
def subtract_start(self, seconds) -> None:
"""
Subtracts <seconds> from startts if startts is set.
:param seconds: Seconds to subtract from starttime
:return: None (Modifies the object in place)
"""
if self.startts:
self.startts = self.startts - seconds
def adjust_start_if_necessary(self, ticker_interval_secs: int, startup_candles: int,
min_date: arrow.Arrow) -> None:
"""
Adjust startts by <startup_candles> candles.
Applies only if no startup-candles have been available.
:param ticker_interval_secs: Ticker interval in seconds e.g. `timeframe_to_seconds('5m')`
:param startup_candles: Number of candles to move start-date forward
:param min_date: Minimum data date loaded. Key kriterium to decide if start-time
has to be moved
:return: None (Modifies the object in place)
"""
if (not self.starttype or (startup_candles
and min_date.timestamp >= self.startts)):
# If no startts was defined, or backtest-data starts at the defined backtest-date
logger.warning("Moving start-date by %s candles to account for startup time.",
startup_candles)
self.startts = (min_date.timestamp + ticker_interval_secs * startup_candles)
self.starttype = 'date'
@staticmethod
def parse_timerange(text: Optional[str]):
"""

52
freqtrade/data/history.py
View File

@@ -8,17 +8,19 @@ Includes:
import logging
import operator
from copy import deepcopy
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import arrow
import pytz
from pandas import DataFrame
from freqtrade import OperationalException, misc
from freqtrade.configuration import TimeRange
from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv
from freqtrade.exchange import Exchange, timeframe_to_minutes
from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_seconds
logger = logging.getLogger(__name__)
@@ -49,6 +51,19 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
return tickerlist[start_index:stop_index]
def trim_dataframe(df: DataFrame, timerange: TimeRange) -> DataFrame:
"""
Trim dataframe based on given timerange
"""
if timerange.starttype == 'date':
start = datetime.fromtimestamp(timerange.startts, tz=pytz.utc)
df = df.loc[df['date'] >= start, :]
if timerange.stoptype == 'date':
stop = datetime.fromtimestamp(timerange.stopts, tz=pytz.utc)
df = df.loc[df['date'] <= stop, :]
return df
def load_tickerdata_file(datadir: Path, pair: str, ticker_interval: str,
timerange: Optional[TimeRange] = None) -> Optional[list]:
"""
@@ -113,7 +128,8 @@ def load_pair_history(pair: str,
refresh_pairs: bool = False,
exchange: Optional[Exchange] = None,
fill_up_missing: bool = True,
drop_incomplete: bool = True
drop_incomplete: bool = True,
startup_candles: int = 0,
) -> DataFrame:
"""
Loads cached ticker history for the given pair.
@@ -126,9 +142,15 @@ def load_pair_history(pair: str,
:param exchange: Exchange object (needed when using "refresh_pairs")
:param fill_up_missing: Fill missing values with "No action"-candles
:param drop_incomplete: Drop last candle assuming it may be incomplete.
:param startup_candles: Additional candles to load at the start of the period
:return: DataFrame with ohlcv data
"""
timerange_startup = deepcopy(timerange)
if startup_candles > 0 and timerange_startup:
logger.info('Using indicator startup period: %s ...', startup_candles)
timerange_startup.subtract_start(timeframe_to_seconds(ticker_interval) * startup_candles)
# The user forced the refresh of pairs
if refresh_pairs:
download_pair_history(datadir=datadir,
@@ -137,11 +159,11 @@ def load_pair_history(pair: str,
ticker_interval=ticker_interval,
timerange=timerange)
pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange_startup)
if pairdata:
if timerange:
_validate_pairdata(pair, pairdata, timerange)
if timerange_startup:
_validate_pairdata(pair, pairdata, timerange_startup)
return parse_ticker_dataframe(pairdata, ticker_interval, pair=pair,
fill_missing=fill_up_missing,
drop_incomplete=drop_incomplete)
@@ -160,10 +182,22 @@ def load_data(datadir: Path,
exchange: Optional[Exchange] = None,
timerange: Optional[TimeRange] = None,
fill_up_missing: bool = True,
startup_candles: int = 0,
fail_without_data: bool = False
) -> Dict[str, DataFrame]:
"""
Loads ticker history data for a list of pairs
:return: dict(<pair>:<tickerlist>)
:param datadir: Path to the data storage location.
:param ticker_interval: Ticker-interval (e.g. "5m")
:param pairs: List of pairs to load
:param refresh_pairs: Refresh pairs from exchange.
(Note: Requires exchange to be passed as well.)
:param exchange: Exchange object (needed when using "refresh_pairs")
:param timerange: Limit data to be loaded to this timerange
:param fill_up_missing: Fill missing values with "No action"-candles
:param startup_candles: Additional candles to load at the start of the period
:param fail_without_data: Raise OperationalException if no data is found.
:return: dict(<pair>:<Dataframe>)
TODO: refresh_pairs is still used by edge to keep the data uptodate.
This should be replaced in the future. Instead, writing the current candles to disk
from dataprovider should be implemented, as this would avoid loading ohlcv data twice.
@@ -176,9 +210,13 @@ def load_data(datadir: Path,
datadir=datadir, timerange=timerange,
refresh_pairs=refresh_pairs,
exchange=exchange,
fill_up_missing=fill_up_missing)
fill_up_missing=fill_up_missing,
startup_candles=startup_candles)
if hist is not None:
result[pair] = hist
if fail_without_data and not result:
raise OperationalException("No data found. Terminating.")
return result

3
freqtrade/edge/__init__.py
View File

@@ -100,7 +100,8 @@ class Edge:
ticker_interval=self.strategy.ticker_interval,
refresh_pairs=self._refresh_pairs,
exchange=self.exchange,
timerange=self._timerange
timerange=self._timerange,
startup_candles=self.strategy.startup_candle_count,
)
if not data:

11
freqtrade/exchange/exchange.py
View File

@@ -228,6 +228,7 @@ class Exchange:
self.validate_pairs(config['exchange']['pair_whitelist'])
self.validate_ordertypes(config.get('order_types', {}))
self.validate_order_time_in_force(config.get('order_time_in_force', {}))
self.validate_required_startup_candles(config.get('startup_candle_count', 0))
# Converts the interval provided in minutes in config to seconds
self.markets_refresh_interval: int = exchange_config.get(
@@ -443,6 +444,16 @@ class Exchange:
raise OperationalException(
f'Time in force policies are not supported for {self.name} yet.')
def validate_required_startup_candles(self, startup_candles) -> None:
"""
Checks if required startup_candles is more than ohlcv_candle_limit.
Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default.
"""
if startup_candles + 5 > self._ft_has['ohlcv_candle_limit']:
raise OperationalException(
f"This strategy requires {startup_candles} candles to start. "
f"{self.name} only provides {self._ft_has['ohlcv_candle_limit']}.")
def exchange_has(self, endpoint: str) -> bool:
"""
Checks if exchange implements a specific API endpoint.

65
freqtrade/optimize/backtesting.py
View File

@@ -15,7 +15,7 @@ from freqtrade import OperationalException
from freqtrade.configuration import TimeRange
from freqtrade.data import history
from freqtrade.data.dataprovider import DataProvider
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.misc import file_dump_json
from freqtrade.persistence import Trade
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -90,6 +90,8 @@ class Backtesting:
self.ticker_interval = str(self.config.get('ticker_interval'))
self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval)
# Get maximum required startup period
self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
# Load one (first) strategy
self._set_strategy(self.strategylist[0])
@@ -103,6 +105,31 @@ class Backtesting:
# And the regular "stoploss" function would not apply to that case
self.strategy.order_types['stoploss_on_exchange'] = False
def load_bt_data(self):
timerange = TimeRange.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
data = history.load_data(
datadir=Path(self.config['datadir']),
pairs=self.config['exchange']['pair_whitelist'],
ticker_interval=self.ticker_interval,
timerange=timerange,
startup_candles=self.required_startup,
fail_without_data=True,
)
min_date, max_date = history.get_timeframe(data)
logger.info(
'Loading data from %s up to %s (%s days)..',
min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
)
# Adjust startts forward if not enough data is available
timerange.adjust_start_if_necessary(timeframe_to_seconds(self.ticker_interval),
self.required_startup, min_date)
return data, timerange
def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame,
skip_nan: bool = False) -> str:
"""
@@ -412,39 +439,18 @@ class Backtesting:
:return: None
"""
data: Dict[str, Any] = {}
pairs = self.config['exchange']['pair_whitelist']
logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
timerange = TimeRange.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
data = history.load_data(
datadir=Path(self.config['datadir']),
pairs=pairs,
ticker_interval=self.ticker_interval,
timerange=timerange,
)
if not data:
logger.critical("No data found. Terminating.")
return
# Use max_open_trades in backtesting, except --disable-max-market-positions is set
if self.config.get('use_max_market_positions', True):
max_open_trades = self.config['max_open_trades']
else:
logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...')
max_open_trades = 0
data, timerange = self.load_bt_data()
all_results = {}
min_date, max_date = history.get_timeframe(data)
logger.info(
'Backtesting with data from %s up to %s (%s days)..',
min_date.isoformat(),
max_date.isoformat(),
(max_date - min_date).days
)
for strat in self.strategylist:
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
self._set_strategy(strat)
@@ -452,6 +458,15 @@ class Backtesting:
# need to reprocess data every time to populate signals
preprocessed = self.strategy.tickerdata_to_dataframe(data)
# Trim startup period from analyzed dataframe
for pair, df in preprocessed.items():
preprocessed[pair] = history.trim_dataframe(df, timerange)
min_date, max_date = history.get_timeframe(preprocessed)
logger.info(
'Backtesting with data from %s up to %s (%s days)..',
min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
)
# Execute backtest and print results
all_results[self.strategy.get_strategy_name()] = self.backtest(
{

26
freqtrade/optimize/hyperopt.py
View File

@@ -22,8 +22,7 @@ from pandas import DataFrame
from skopt import Optimizer
from skopt.space import Dimension
from freqtrade.configuration import TimeRange
from freqtrade.data.history import load_data, get_timeframe
from freqtrade.data.history import get_timeframe, trim_dataframe
from freqtrade.misc import round_dict
from freqtrade.optimize.backtesting import Backtesting
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
@@ -379,30 +378,19 @@ class Hyperopt:
)
def start(self) -> None:
timerange = TimeRange.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
data = load_data(
datadir=Path(self.config['datadir']),
pairs=self.config['exchange']['pair_whitelist'],
ticker_interval=self.backtesting.ticker_interval,
timerange=timerange
)
data, timerange = self.backtesting.load_bt_data()
if not data:
logger.critical("No data found. Terminating.")
return
preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data)
# Trim startup period from analyzed dataframe
for pair, df in preprocessed.items():
preprocessed[pair] = trim_dataframe(df, timerange)
min_date, max_date = get_timeframe(data)
logger.info(
'Hyperopting with data from %s up to %s (%s days)..',
min_date.isoformat(),
max_date.isoformat(),
(max_date - min_date).days
min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days
)
preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data)
dump(preprocessed, self.tickerdata_pickle)
# We don't need exchange instance anymore while running hyperopt

1
freqtrade/resolvers/strategy_resolver.py
View File

@@ -57,6 +57,7 @@ class StrategyResolver(IResolver):
("order_time_in_force", None, False),
("stake_currency", None, False),
("stake_amount", None, False),
("startup_candle_count", None, False),
("use_sell_signal", True, True),
("sell_profit_only", False, True),
("ignore_roi_if_buy_signal", False, True),

6
freqtrade/strategy/default_strategy.py
View File

@@ -39,6 +39,9 @@ class DefaultStrategy(IStrategy):
'stoploss_on_exchange': False
}
# Number of candles the strategy requires before producing valid signals
startup_candle_count: int = 20
# Optional time in force for orders
order_time_in_force = {
'buy': 'gtc',
@@ -105,9 +108,6 @@ class DefaultStrategy(IStrategy):
# EMA - Exponential Moving Average
dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10)
# SMA - Simple Moving Average
dataframe['sma'] = ta.SMA(dataframe, timeperiod=40)
return dataframe
def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

4
freqtrade/strategy/interface.py
View File

@@ -103,6 +103,9 @@ class IStrategy(ABC):
# run "populate_indicators" only for new candle
process_only_new_candles: bool = False
# Count of candles the strategy requires before producing valid signals
startup_candle_count: int = 0
# Class level variables (intentional) containing
# the dataprovider (dp) (access to other candles, historic data, ...)
# and wallets - access to the current balance.
@@ -421,6 +424,7 @@ class IStrategy(ABC):
def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]:
"""
Creates a dataframe and populates indicators for given ticker data
Used by optimize operations only, not during dry / live runs.
"""
return {pair: self.advise_indicators(pair_data, {'pair': pair})
for pair, pair_data in tickerdata.items()}