adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature-unlimited-stake_amount

Browse Source
This commit is contained in:
Anton
2018-06-17 02:23:40 +03:00
parent eb909068c5 934974a547
commit ae94ab17f4
31 changed files with 652 additions and 907 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
freqtrade/optimize/__init__.py
View File

@@ -11,8 +11,6 @@ from freqtrade import misc, constants
from freqtrade.exchange import get_ticker_history
from freqtrade.arguments import TimeRange
from user_data.hyperopt_conf import hyperopt_optimize_conf
logger = logging.getLogger(__name__)
@@ -83,7 +81,7 @@ def load_tickerdata_file(
def load_data(datadir: str,
ticker_interval: str,
pairs: Optional[List[str]] = None,
pairs: List[str],
refresh_pairs: Optional[bool] = False,
timerange: TimeRange = TimeRange(None, None, 0, 0)) -> Dict[str, List]:
"""
@@ -92,14 +90,12 @@ def load_data(datadir: str,
"""
result = {}
_pairs = pairs or hyperopt_optimize_conf()['exchange']['pair_whitelist']
# If the user force the refresh of pairs
if refresh_pairs:
logger.info('Download data for all pairs and store them in %s', datadir)
download_pairs(datadir, _pairs, ticker_interval, timerange=timerange)
download_pairs(datadir, pairs, ticker_interval, timerange=timerange)
for pair in _pairs:
for pair in pairs:
pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
if pairdata:
result[pair] = pairdata

151
freqtrade/optimize/backtesting.py
View File

@@ -6,7 +6,8 @@ This module contains the backtesting logic
import logging
import operator
from argparse import Namespace
from typing import Dict, Tuple, Any, List, Optional
from datetime import datetime
from typing import Dict, Tuple, Any, List, Optional, NamedTuple
import arrow
from pandas import DataFrame
@@ -23,6 +24,21 @@ from freqtrade.persistence import Trade
logger = logging.getLogger(__name__)
class BacktestResult(NamedTuple):
"""
NamedTuple Defining BacktestResults inputs.
"""
pair: str
profit_percent: float
profit_abs: float
open_time: datetime
close_time: datetime
open_index: int
close_index: int
trade_duration: float
open_at_end: bool
class Backtesting(object):
"""
Backtesting class, this class contains all the logic to run a backtest
@@ -73,15 +89,15 @@ class Backtesting(object):
headers = ['pair', 'buy count', 'avg profit %',
'total profit ' + stake_currency, 'avg duration', 'profit', 'loss']
for pair in data:
result = results[results.currency == pair]
result = results[results.pair == pair]
tabular_data.append([
pair,
len(result.index),
result.profit_percent.mean() * 100.0,
result.profit_BTC.sum(),
result.duration.mean(),
len(result[result.profit_BTC > 0]),
len(result[result.profit_BTC < 0])
result.profit_abs.sum(),
result.trade_duration.mean(),
len(result[result.profit_abs > 0]),
len(result[result.profit_abs < 0])
])
# Append Total
@@ -89,16 +105,28 @@ class Backtesting(object):
'TOTAL',
len(results.index),
results.profit_percent.mean() * 100.0,
results.profit_BTC.sum(),
results.duration.mean(),
len(results[results.profit_BTC > 0]),
len(results[results.profit_BTC < 0])
results.profit_abs.sum(),
results.trade_duration.mean(),
len(results[results.profit_abs > 0]),
len(results[results.profit_abs < 0])
])
return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe")
def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None:
records = [(trade_entry.pair, trade_entry.profit_percent,
trade_entry.open_time.timestamp(),
trade_entry.close_time.timestamp(),
trade_entry.open_index - 1, trade_entry.trade_duration)
for index, trade_entry in results.iterrows()]
if records:
logger.info('Dumping backtest results to %s', recordfilename)
file_dump_json(recordfilename, records)
def _get_sell_trade_entry(
self, pair: str, buy_row: DataFrame,
partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[Tuple]:
partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[BacktestResult]:
stake_amount = args['stake_amount']
max_open_trades = args.get('max_open_trades', 0)
@@ -121,15 +149,33 @@ class Backtesting(object):
buy_signal = sell_row.buy
if self.analyze.should_sell(trade, sell_row.close, sell_row.date, buy_signal,
sell_row.sell):
return \
sell_row, \
(
pair,
trade.calc_profit_percent(rate=sell_row.close),
trade.calc_profit(rate=sell_row.close),
(sell_row.date - buy_row.date).seconds // 60
), \
sell_row.date
return BacktestResult(pair=pair,
profit_percent=trade.calc_profit_percent(rate=sell_row.close),
profit_abs=trade.calc_profit(rate=sell_row.close),
open_time=buy_row.date,
close_time=sell_row.date,
trade_duration=(sell_row.date - buy_row.date).seconds // 60,
open_index=buy_row.Index,
close_index=sell_row.Index,
open_at_end=False
)
if partial_ticker:
# no sell condition found - trade stil open at end of backtest period
sell_row = partial_ticker[-1]
btr = BacktestResult(pair=pair,
profit_percent=trade.calc_profit_percent(rate=sell_row.close),
profit_abs=trade.calc_profit(rate=sell_row.close),
open_time=buy_row.date,
close_time=sell_row.date,
trade_duration=(sell_row.date - buy_row.date).seconds // 60,
open_index=buy_row.Index,
close_index=sell_row.Index,
open_at_end=True
)
logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair,
btr.profit_percent, btr.profit_abs)
return btr
return None
def backtest(self, args: Dict) -> DataFrame:
@@ -145,17 +191,12 @@ class Backtesting(object):
processed: a processed dictionary with format {pair, data}
max_open_trades: maximum number of concurrent trades (default: 0, disabled)
realistic: do we try to simulate realistic trades? (default: True)
sell_profit_only: sell if profit only
use_sell_signal: act on sell-signal
:return: DataFrame
"""
headers = ['date', 'buy', 'open', 'close', 'sell']
processed = args['processed']
max_open_trades = args.get('max_open_trades', 0)
realistic = args.get('realistic', False)
record = args.get('record', None)
recordfilename = args.get('recordfn', 'backtest-result.json')
records = []
trades = []
trade_count_lock: Dict = {}
for pair, pair_data in processed.items():
@@ -170,6 +211,8 @@ class Backtesting(object):
ticker_data.drop(ticker_data.head(1).index, inplace=True)
# Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.)
ticker = [x for x in ticker_data.itertuples()]
lock_pair_until = None
@@ -187,28 +230,18 @@ class Backtesting(object):
trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
ret = self._get_sell_trade_entry(pair, row, ticker[index + 1:],
trade_count_lock, args)
trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:],
trade_count_lock, args)
if ret:
row2, trade_entry, next_date = ret
lock_pair_until = next_date
if trade_entry:
lock_pair_until = trade_entry.close_time
trades.append(trade_entry)
if record:
# Note, need to be json.dump friendly
# record a tuple of pair, current_profit_percent,
# entry-date, duration
records.append((pair, trade_entry[1],
row.date.strftime('%s'),
row2.date.strftime('%s'),
index, trade_entry[3]))
# For now export inside backtest(), maybe change so that backtest()
# returns a tuple like: (dataframe, records, logs, etc)
if record and record.find('trades') >= 0:
logger.info('Dumping backtest results to %s', recordfilename)
file_dump_json(recordfilename, records)
labels = ['currency', 'profit_percent', 'profit_BTC', 'duration']
return DataFrame.from_records(trades, columns=labels)
else:
# Set lock_pair_until to end of testing period if trade could not be closed
# This happens only if the buy-signal was with the last candle
lock_pair_until = ticker_data.iloc[-1].date
return DataFrame.from_records(trades, columns=BacktestResult._fields)
def start(self) -> None:
"""
@@ -237,6 +270,9 @@ class Backtesting(object):
timerange=timerange
)
if not data:
logger.critical("No data found. Terminating.")
return
# Ignore max_open_trades in backtesting, except realistic flag was passed
if self.config.get('realistic_simulation', False):
max_open_trades = self.config['max_open_trades']
@@ -256,24 +292,22 @@ class Backtesting(object):
)
# Execute backtest and print results
sell_profit_only = self.config.get('experimental', {}).get('sell_profit_only', False)
use_sell_signal = self.config.get('experimental', {}).get('use_sell_signal', False)
results = self.backtest(
{
'stake_amount': self.config.get('stake_amount'),
'processed': preprocessed,
'max_open_trades': max_open_trades,
'realistic': self.config.get('realistic_simulation', False),
'sell_profit_only': sell_profit_only,
'use_sell_signal': use_sell_signal,
'record': self.config.get('export'),
'recordfn': self.config.get('exportfilename'),
}
)
if self.config.get('export', False):
self._store_backtest_result(self.config.get('exportfilename'), results)
logger.info(
'\n==================================== '
'\n======================================== '
'BACKTESTING REPORT'
' ====================================\n'
' =========================================\n'
'%s',
self._generate_text_table(
data,
@@ -281,6 +315,17 @@ class Backtesting(object):
)
)
logger.info(
'\n====================================== '
'LEFT OPEN TRADES REPORT'
' ======================================\n'
'%s',
self._generate_text_table(
data,
results.loc[results.open_at_end]
)
)
def setup_configuration(args: Namespace) -> Dict[str, Any]:
"""

48
freqtrade/optimize/hyperopt.py
View File

@@ -19,7 +19,6 @@ from typing import Dict, Any, Callable, Optional
import numpy
import talib.abstract as ta
from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe
from hyperopt.mongoexp import MongoTrials
from pandas import DataFrame
import freqtrade.vendor.qtpylib.indicators as qtpylib
@@ -27,7 +26,6 @@ from freqtrade.arguments import Arguments
from freqtrade.configuration import Configuration
from freqtrade.optimize import load_data
from freqtrade.optimize.backtesting import Backtesting
from user_data.hyperopt_conf import hyperopt_optimize_conf
logger = logging.getLogger(__name__)
@@ -451,7 +449,7 @@ class Hyperopt(Backtesting):
total_profit = results.profit_percent.sum()
trade_count = len(results.index)
trade_duration = results.duration.mean()
trade_duration = results.trade_duration.mean()
if trade_count == 0 or trade_duration > self.max_accepted_trade_duration:
print('.', end='')
@@ -488,10 +486,10 @@ class Hyperopt(Backtesting):
'Total profit {: 11.8f} {} ({:.4f}Σ%). Avg duration {:5.1f} mins.').format(
len(results.index),
results.profit_percent.mean() * 100.0,
results.profit_BTC.sum(),
results.profit_abs.sum(),
self.config['stake_currency'],
results.profit_percent.sum(),
results.duration.mean(),
results.trade_duration.mean(),
)
def start(self) -> None:
@@ -508,32 +506,20 @@ class Hyperopt(Backtesting):
self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore
self.processed = self.tickerdata_to_dataframe(data)
if self.config.get('mongodb'):
logger.info('Using mongodb ...')
logger.info('Preparing Trials..')
signal.signal(signal.SIGINT, self.signal_handler)
# read trials file if we have one
if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0:
self.trials = self.read_trials()
self.current_tries = len(self.trials.results)
self.total_tries += self.current_tries
logger.info(
'Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!'
'Continuing with trials. Current: %d, Total: %d',
self.current_tries,
self.total_tries
)
db_name = 'freqtrade_hyperopt'
self.trials = MongoTrials(
arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name),
exp_key='exp1'
)
else:
logger.info('Preparing Trials..')
signal.signal(signal.SIGINT, self.signal_handler)
# read trials file if we have one
if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0:
self.trials = self.read_trials()
self.current_tries = len(self.trials.results)
self.total_tries += self.current_tries
logger.info(
'Continuing with trials. Current: %d, Total: %d',
self.current_tries,
self.total_tries
)
try:
best_parameters = fmin(
fn=self.generate_optimizer,
@@ -589,18 +575,14 @@ def start(args: Namespace) -> None:
"""
# Remove noisy log messages
logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING)
logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
# Initialize configuration
# Monkey patch the configuration with hyperopt_conf.py
configuration = Configuration(args)
logger.info('Starting freqtrade in Hyperopt mode')
config = configuration.load_config()
optimize_config = hyperopt_optimize_conf()
config = configuration._load_common_config(optimize_config)
config = configuration._load_backtesting_config(config)
config = configuration._load_hyperopt_config(config)
config['exchange']['key'] = ''
config['exchange']['secret'] = ''