adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into support_multiple_ticker

Browse Source
This commit is contained in:
Jean-Baptiste LE STANG
2018-01-20 19:25:47 +01:00
parent c9e1fd3fc4 a7e561b55f
commit 36797cda30
22 changed files with 767 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
freqtrade/optimize/__init__.py
View File

@@ -8,11 +8,25 @@ from pandas import DataFrame
from freqtrade.exchange import get_ticker_history
from freqtrade.optimize.hyperopt_conf import hyperopt_optimize_conf
from freqtrade.analyze import populate_indicators, parse_ticker_dataframe
from freqtrade import misc
logger = logging.getLogger(__name__)
def load_tickerdata_file(datadir, pair, ticker_interval):
def trim_tickerlist(tickerlist, timerange):
(stype, start, stop) = timerange
if stype == (None, 'line'):
return tickerlist[stop:]
elif stype == ('line', None):
return tickerlist[0:start]
elif stype == ('index', 'index'):
return tickerlist[start:stop]
else:
return tickerlist
def load_tickerdata_file(datadir, pair, ticker_interval,
timerange=None):
"""
Load a pair from file,
:return dict OR empty if unsuccesful
@@ -30,11 +44,13 @@ def load_tickerdata_file(datadir, pair, ticker_interval):
# Read the file, load the json
with open(file) as tickerdata:
pairdata = json.load(tickerdata)
if timerange:
pairdata = trim_tickerlist(pairdata, timerange)
return pairdata
def load_data(datadir: str, ticker_interval: int, pairs: Optional[List[str]] = None,
refresh_pairs: Optional[bool] = False) -> Dict[str, List]:
refresh_pairs: Optional[bool] = False, timerange=None) -> Dict[str, List]:
"""
Loads ticker history data for the given parameters
:param ticker_interval: ticker interval in minutes
@@ -51,16 +67,21 @@ def load_data(datadir: str, ticker_interval: int, pairs: Optional[List[str]] = N
download_pairs(datadir, _pairs, ticker_interval)
for pair in _pairs:
pairdata = load_tickerdata_file(datadir, pair, ticker_interval)
pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
if not pairdata:
# download the tickerdata from exchange
download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval)
# and retry reading the pair
pairdata = load_tickerdata_file(datadir, pair, ticker_interval)
pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
result[pair] = pairdata
return result
def tickerdata_to_dataframe(data):
preprocessed = preprocess(data)
return preprocessed
def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]:
"""Creates a dataframe and populates indicators for given ticker data"""
return {pair: populate_indicators(parse_ticker_dataframe(pair_data))
@@ -126,7 +147,6 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) ->
logger.debug("New End: {}".format(data[-1:][0]['T']))
data = sorted(data, key=lambda data: data['T'])
with open(filename, "wt") as fp:
json.dump(data, fp)
misc.file_dump_json(filename, data)
return True

139
freqtrade/optimize/backtesting.py
View File

@@ -13,7 +13,6 @@ from freqtrade import exchange
from freqtrade.analyze import populate_buy_trend, populate_sell_trend
from freqtrade.exchange import Bittrex
from freqtrade.main import min_roi_reached
from freqtrade.optimize import preprocess
from freqtrade.persistence import Trade
logger = logging.getLogger(__name__)
@@ -67,17 +66,60 @@ def generate_text_table(
return tabulate(tabular_data, headers=headers, floatfmt=floatfmt)
def backtest(stake_amount: float, processed: Dict[str, DataFrame],
max_open_trades: int = 0, realistic: bool = True, sell_profit_only: bool = False,
stoploss: int = -1.00, use_sell_signal: bool = False) -> DataFrame:
def get_trade_entry(pair, row, ticker, trade_count_lock, args):
stake_amount = args['stake_amount']
max_open_trades = args.get('max_open_trades', 0)
sell_profit_only = args.get('sell_profit_only', False)
stoploss = args.get('stoploss', -1)
use_sell_signal = args.get('use_sell_signal', False)
trade = Trade(open_rate=row.close,
open_date=row.date,
stake_amount=stake_amount,
amount=stake_amount / row.open,
fee=exchange.get_fee()
)
# calculate win/lose forwards from buy point
sell_subset = ticker[row.Index + 1:][['close', 'date', 'sell']]
for row2 in sell_subset.itertuples(index=True):
if max_open_trades > 0:
# Increase trade_count_lock for every iteration
trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1
current_profit_percent = trade.calc_profit_percent(rate=row2.close)
if (sell_profit_only and current_profit_percent < 0):
continue
if min_roi_reached(trade, row2.close, row2.date) or \
(row2.sell == 1 and use_sell_signal) or \
current_profit_percent <= stoploss:
current_profit_btc = trade.calc_profit(rate=row2.close)
return row2, (pair,
current_profit_percent,
current_profit_btc,
row2.Index - row.Index,
current_profit_btc > 0,
current_profit_btc < 0
)
def backtest(args) -> DataFrame:
"""
Implements backtesting functionality
:param stake_amount: btc amount to use for each trade
:param processed: a processed dictionary with format {pair, data}
:param max_open_trades: maximum number of concurrent trades (default: 0, disabled)
:param realistic: do we try to simulate realistic trades? (default: True)
:param args: a dict containing:
stake_amount: btc amount to use for each trade
processed: a processed dictionary with format {pair, data}
max_open_trades: maximum number of concurrent trades (default: 0, disabled)
realistic: do we try to simulate realistic trades? (default: True)
sell_profit_only: sell if profit only
use_sell_signal: act on sell-signal
stoploss: use stoploss
:return: DataFrame
"""
processed = args['processed']
max_open_trades = args.get('max_open_trades', 0)
realistic = args.get('realistic', True)
record = args.get('record', None)
records = []
trades = []
trade_count_lock: dict = {}
exchange._API = Bittrex({'key': '', 'secret': ''})
@@ -100,41 +142,25 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame],
# Increase lock
trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
trade = Trade(
open_rate=row.close,
open_date=row.date,
stake_amount=stake_amount,
amount=stake_amount / row.open,
fee=exchange.get_fee()
)
# calculate win/lose forwards from buy point
sell_subset = ticker[row.Index + 1:][['close', 'date', 'sell']]
for row2 in sell_subset.itertuples(index=True):
if max_open_trades > 0:
# Increase trade_count_lock for every iteration
trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1
current_profit_percent = trade.calc_profit_percent(rate=row2.close)
if (sell_profit_only and current_profit_percent < 0):
continue
if min_roi_reached(trade, row2.close, row2.date) or \
(row2.sell == 1 and use_sell_signal) or \
current_profit_percent <= stoploss:
current_profit_btc = trade.calc_profit(rate=row2.close)
lock_pair_until = row2.Index
trades.append(
(
pair,
current_profit_percent,
current_profit_btc,
row2.Index - row.Index,
current_profit_btc > 0,
current_profit_btc < 0
)
)
break
ret = get_trade_entry(pair, row, ticker,
trade_count_lock, args)
if ret:
row2, trade_entry = ret
lock_pair_until = row2.Index
trades.append(trade_entry)
if record:
# Note, need to be json.dump friendly
# record a tuple of pair, current_profit_percent,
# entry-date, duration
records.append((pair, trade_entry[1],
row.date.strftime('%s'),
row2.date.strftime('%s'),
row.Index, trade_entry[3]))
# For now export inside backtest(), maybe change so that backtest()
# returns a tuple like: (dataframe, records, logs, etc)
if record and record.find('trades') >= 0:
logger.info('Dumping backtest results')
misc.file_dump_json('backtest-result.json', records)
labels = ['currency', 'profit_percent', 'profit_BTC', 'duration', 'profit', 'loss']
return DataFrame.from_records(trades, columns=labels)
@@ -167,6 +193,10 @@ def start(args):
logger.info('Using stake_currency: %s ...', config['stake_currency'])
logger.info('Using stake_amount: %s ...', config['stake_amount'])
timerange = misc.parse_timerange(args.timerange)
data = optimize.load_data(args.datadir, pairs=pairs, ticker_interval=args.ticker_interval,
refresh_pairs=args.refresh_pairs,
timerange=timerange)
max_open_trades = 0
if args.realistic_simulation:
logger.info('Using max_open_trades: %s ...', config['max_open_trades'])
@@ -176,21 +206,22 @@ def start(args):
from freqtrade import main
main._CONF = config
preprocessed = preprocess(data)
preprocessed = optimize.tickerdata_to_dataframe(data)
# Print timeframe
min_date, max_date = get_timeframe(preprocessed)
logger.info('Measuring data from %s up to %s ...', min_date.isoformat(), max_date.isoformat())
# Execute backtest and print results
results = backtest(
stake_amount=config['stake_amount'],
processed=preprocessed,
max_open_trades=max_open_trades,
realistic=args.realistic_simulation,
sell_profit_only=config.get('experimental', {}).get('sell_profit_only', False),
stoploss=config.get('stoploss'),
use_sell_signal=config.get('experimental', {}).get('use_sell_signal', False)
)
sell_profit_only = config.get('experimental', {}).get('sell_profit_only', False)
use_sell_signal = config.get('experimental', {}).get('use_sell_signal', False)
results = backtest({'stake_amount': config['stake_amount'],
'processed': preprocessed,
'max_open_trades': max_open_trades,
'realistic': args.realistic_simulation,
'sell_profit_only': sell_profit_only,
'use_sell_signal': use_sell_signal,
'stoploss': config.get('stoploss'),
'record': args.export
})
logger.info(
'\n==================================== BACKTESTING REPORT ====================================\n%s', # noqa
generate_text_table(data, results, config['stake_currency'], args.ticker_interval)

13
freqtrade/optimize/hyperopt.py
View File

@@ -15,7 +15,7 @@ from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe
from hyperopt.mongoexp import MongoTrials
from pandas import DataFrame
from freqtrade import main # noqa
from freqtrade import main, misc # noqa
from freqtrade import exchange, optimize
from freqtrade.exchange import Bittrex
from freqtrade.misc import load_config
@@ -164,7 +164,9 @@ def optimizer(params):
from freqtrade.optimize import backtesting
backtesting.populate_buy_trend = buy_strategy_generator(params)
results = backtest(OPTIMIZE_CONFIG['stake_amount'], PROCESSED, stoploss=params['stoploss'])
results = backtest({'stake_amount': OPTIMIZE_CONFIG['stake_amount'],
'processed': PROCESSED,
'stoploss': params['stoploss']})
result_explanation = format_results(results)
total_profit = results.profit_percent.sum()
@@ -273,8 +275,11 @@ def start(args):
logger.info('Using config: %s ...', args.config)
config = load_config(args.config)
pairs = config['exchange']['pair_whitelist']
PROCESSED = optimize.preprocess(optimize.load_data(
args.datadir, pairs=pairs, ticker_interval=args.ticker_interval))
timerange = misc.parse_timerange(args.timerange)
data = optimize.load_data(args.datadir, pairs=pairs,
ticker_interval=args.ticker_interval,
timerange=timerange)
PROCESSED = optimize.tickerdata_to_dataframe(data)
if args.mongodb:
logger.info('Using mongodb ...')