adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix crash when backtest-result.json not exist

Browse Source
This commit is contained in:
AxelCh 2019-01-23 14:11:05 -04:00 committed by Matthias
parent 06e0616fb0
commit eec7276393
5 changed files with 473 additions and 389 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
docs/plotting.md
View File

@ -15,7 +15,7 @@ At least version 2.3.0 is required.
Usage for the price plotter: Usage for the price plotter:
``` ```
script/plot_dataframe.py [-h] [-p pair] [--live] script/plot_dataframe.py [-h] [-p pairs] [--live]
``` ```
Example Example
@ -23,11 +23,16 @@ Example
python scripts/plot_dataframe.py -p BTC/ETH python scripts/plot_dataframe.py -p BTC/ETH
``` ```
The `-p` pair argument, can be used to specify what The `-p` pairs argument, can be used to specify
pair you would like to plot. pairs you would like to plot.
**Advanced use** **Advanced use**
To plot multiple pairs, separate them with a comma:
```
python scripts/plot_dataframe.py -p BTC/ETH,XRP/ETH
```
To plot the current live price use the `--live` flag: To plot the current live price use the `--live` flag:
``` ```
python scripts/plot_dataframe.py -p BTC/ETH --live python scripts/plot_dataframe.py -p BTC/ETH --live

4
freqtrade/arguments.py
View File

@ -352,9 +352,9 @@ class Arguments(object):
Parses given arguments for scripts. Parses given arguments for scripts.
""" """
self.parser.add_argument( self.parser.add_argument(
'-p', '--pair', '-p', '--pairs',
help='Show profits for only this pairs. Pairs are comma-separated.', help='Show profits for only this pairs. Pairs are comma-separated.',
dest='pair', dest='pairs',
default=None default=None
) )

2
freqtrade/tests/test_arguments.py
View File

@ -47,7 +47,7 @@ def test_scripts_options() -> None:
arguments = Arguments(['-p', 'ETH/BTC'], '') arguments = Arguments(['-p', 'ETH/BTC'], '')
arguments.scripts_options() arguments.scripts_options()
args = arguments.get_parsed_arg() args = arguments.get_parsed_arg()
assert args.pair == 'ETH/BTC' assert args.pairs == 'ETH/BTC'
def test_parse_args_version() -> None: def test_parse_args_version() -> None:

841
scripts/plot_dataframe.py
View File

@ -1,381 +1,460 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
Script to display when the bot will buy a specific pair Script to display when the bot will buy on specific pair(s)
Mandatory Cli parameters: Mandatory Cli parameters:
-p / --pair: pair to examine -p / --pairs: pair(s) to examine
Option but recommended Option but recommended
-s / --strategy: strategy to use -s / --strategy: strategy to use
Optional Cli parameters Optional Cli parameters
-d / --datadir: path to pair backtest data -d / --datadir: path to pair(s) backtest data
--timerange: specify what timerange of data to use. --timerange: specify what timerange of data to use.
-l / --live: Live, to download the latest ticker for the pair -l / --live: Live, to download the latest ticker for the pair(s)
-db / --db-url: Show trades stored in database -db / --db-url: Show trades stored in database
Indicators recommended Indicators recommended
Row 1: sma, ema3, ema5, ema10, ema50 Row 1: sma, ema3, ema5, ema10, ema50
Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk
Example of usage: Example of usage:
> python3 scripts/plot_dataframe.py --pair BTC/EUR -d user_data/data/ --indicators1 sma,ema3 > python3 scripts/plot_dataframe.py --pairs BTC/EUR,XRP/BTC -d user_data/data/ --indicators1 sma,ema3
--indicators2 fastk,fastd --indicators2 fastk,fastd
""" """
import json import json
import logging import logging
import sys import sys
from argparse import Namespace import os
from pathlib import Path from argparse import Namespace
from typing import Dict, List, Any from pathlib import Path
from typing import Dict, List, Any
import pandas as pd
import plotly.graph_objs as go import pandas as pd
import pytz import plotly.graph_objs as go
import pytz
from plotly import tools
from plotly.offline import plot from plotly import tools
from plotly.offline import plot
from freqtrade import persistence
from freqtrade.arguments import Arguments, TimeRange from freqtrade import persistence
from freqtrade.data import history from freqtrade.arguments import Arguments, TimeRange
from freqtrade.exchange import Exchange from freqtrade.data import history
from freqtrade.optimize.backtesting import setup_configuration from freqtrade.exchange import Exchange
from freqtrade.persistence import Trade from freqtrade.optimize.backtesting import setup_configuration
from freqtrade.resolvers import StrategyResolver from freqtrade.persistence import Trade
from freqtrade.resolvers import StrategyResolver
logger = logging.getLogger(__name__)
_CONF: Dict[str, Any] = {} logger = logging.getLogger(__name__)
_CONF: Dict[str, Any] = {}
timeZone = pytz.UTC
timeZone = pytz.UTC
def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame:
trades: pd.DataFrame = pd.DataFrame() def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame:
if args.db_url: trades: pd.DataFrame = pd.DataFrame()
persistence.init(_CONF) if args.db_url:
columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] persistence.init(_CONF)
columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"]
for x in Trade.query.all():
print("date: {}".format(x.open_date)) for x in Trade.query.all():
print("date: {}".format(x.open_date))
trades = pd.DataFrame([(t.pair, t.calc_profit(),
t.open_date.replace(tzinfo=timeZone), trades = pd.DataFrame([(t.pair, t.calc_profit(),
t.close_date.replace(tzinfo=timeZone) if t.close_date else None, t.open_date.replace(tzinfo=timeZone),
t.open_rate, t.close_rate, t.close_date.replace(tzinfo=timeZone) if t.close_date else None,
t.close_date.timestamp() - t.open_date.timestamp() if t.close_date else None) t.open_rate, t.close_rate,
for t in Trade.query.filter(Trade.pair.is_(pair)).all()], t.close_date.timestamp() - t.open_date.timestamp()
columns=columns) if t.close_date else None)
for t in Trade.query.filter(Trade.pair.is_(pair)).all()],
elif args.exportfilename: columns=columns)
file = Path(args.exportfilename)
# must align with columns in backtest.py elif args.exportfilename:
columns = ["pair", "profit", "opents", "closets", "index", "duration", file = Path(args.exportfilename)
"open_rate", "close_rate", "open_at_end", "sell_reason"] # must align with columns in backtest.py
with file.open() as f: columns = ["pair", "profit", "opents", "closets", "index", "duration",
data = json.load(f) "open_rate", "close_rate", "open_at_end", "sell_reason"]
trades = pd.DataFrame(data, columns=columns) if os.path.exists(file):
trades = trades.loc[trades["pair"] == pair] with file.open() as f:
if timerange: data = json.load(f)
if timerange.starttype == 'date': trades = pd.DataFrame(data, columns=columns)
trades = trades.loc[trades["opents"] >= timerange.startts] trades = trades.loc[trades["pair"] == pair]
if timerange.stoptype == 'date': if timerange:
trades = trades.loc[trades["opents"] <= timerange.stopts] if timerange.starttype == 'date':
trades = trades.loc[trades["opents"] >= timerange.startts]
trades['opents'] = pd.to_datetime(trades['opents'], if timerange.stoptype == 'date':
unit='s', trades = trades.loc[trades["opents"] <= timerange.stopts]
utc=True,
infer_datetime_format=True) trades['opents'] = pd.to_datetime(
trades['closets'] = pd.to_datetime(trades['closets'], trades['opents'],
unit='s', unit='s',
utc=True, utc=True,
infer_datetime_format=True) infer_datetime_format=True)
return trades trades['closets'] = pd.to_datetime(
trades['closets'],
unit='s',
def plot_analyzed_dataframe(args: Namespace) -> None: utc=True,
""" infer_datetime_format=True)
Calls analyze() and plots the returned dataframe else:
:return: None trades = pd.DataFrame([], columns=columns)
"""
global _CONF return trades
# Load the configuration
_CONF.update(setup_configuration(args)) def generate_plot_file(fig, pair, tick_interval, is_last) -> None:
"""
print(_CONF) Generate a plot html file from pre populated fig plotly object
# Set the pair to audit :return: None
pair = args.pair """
logger.info('Generate plot file for %s', pair)
if pair is None:
logger.critical('Parameter --pair mandatory;. E.g --pair ETH/BTC') pair_name = pair.replace("/", "_")
exit() file_name = 'freqtrade-plot-' + pair_name + '-' + tick_interval + '.html'
if '/' not in pair: if not os.path.exists('user_data/plots'):
logger.critical('--pair format must be XXX/YYY') try:
exit() os.makedirs('user_data/plots')
except OSError as e:
# Set timerange to use raise
timerange = Arguments.parse_timerange(args.timerange)
plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), auto_open=False)
# Load the strategy if is_last:
try: plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')), auto_open=False)
strategy = StrategyResolver(_CONF).strategy
exchange = Exchange(_CONF)
except AttributeError: def get_trading_env(args: Namespace):
logger.critical( """
'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', Initalize freqtrade Exchange and Strategy, split pairs recieved in parameter
args.strategy :return: Strategy
) """
exit() global _CONF
# Set the ticker to use # Load the configuration
tick_interval = strategy.ticker_interval _CONF.update(setup_configuration(args))
print(_CONF)
# Load pair tickers
tickers = {} pairs = args.pairs.split(',')
if args.live: if pairs is None:
logger.info('Downloading pair.') logger.critical('Parameter --pairs mandatory;. E.g --pairs ETH/BTC,XRP/BTC')
exchange.refresh_tickers([pair], tick_interval) exit()
tickers[pair] = exchange.klines(pair)
else: # Load the strategy
tickers = history.load_data( try:
datadir=Path(_CONF.get("datadir")), strategy = StrategyResolver(_CONF).strategy
pairs=[pair], exchange = Exchange(_CONF)
ticker_interval=tick_interval, except AttributeError:
refresh_pairs=_CONF.get('refresh_pairs', False), logger.critical(
timerange=timerange, 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"',
exchange=Exchange(_CONF) args.strategy
) )
exit()
# No ticker found, or impossible to download
if tickers == {}: return [strategy, exchange, pairs]
exit()
# Get trades already made from the DB def get_tickers_data(strategy, exchange, pairs: List[str], args):
trades = load_trades(args, pair, timerange) """
Get tickers data for each pairs on live or local, option defined in args
dataframes = strategy.tickerdata_to_dataframe(tickers) :return: dictinnary of tickers. output format: {'pair': tickersdata}
"""
dataframe = dataframes[pair]
dataframe = strategy.advise_buy(dataframe, {'pair': pair}) tick_interval = strategy.ticker_interval
dataframe = strategy.advise_sell(dataframe, {'pair': pair}) timerange = Arguments.parse_timerange(args.timerange)
if len(dataframe.index) > args.plot_limit: tickers = {}
logger.warning('Ticker contained more than %s candles as defined ' if args.live:
'with --plot-limit, clipping.', args.plot_limit) logger.info('Downloading pairs.')
dataframe = dataframe.tail(args.plot_limit) exchange.refresh_tickers(pairs, tick_interval)
for pair in pairs:
trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']] tickers[pair] = exchange.klines(pair)
fig = generate_graph( else:
pair=pair, tickers = history.load_data(
trades=trades, datadir=Path(_CONF.get("datadir")),
data=dataframe, pairs=pairs,
args=args ticker_interval=tick_interval,
) refresh_pairs=_CONF.get('refresh_pairs', False),
timerange=timerange,
plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html'))) exchange=Exchange(_CONF)
)
def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: # No ticker found, impossible to download, len mismatch
""" for pair, data in tickers.copy().items():
Generate the graph from the data generated by Backtesting or from DB logger.debug("checking tickers data of pair: %s", pair)
:param pair: Pair to Display on the graph logger.debug("data.empty: %s", data.empty)
:param trades: All trades created logger.debug("len(data): %s", len(data))
:param data: Dataframe if data.empty:
:param args: sys.argv that contrains the two params indicators1, and indicators2 del tickers[pair]
:return: None logger.info(
""" 'An issue occured while retreiving datas of %s pair, please retry '
'using -l option for live or --refresh-pairs-cached', pair)
# Define the graph return tickers
fig = tools.make_subplots(
rows=3,
cols=1, def generate_dataframe(strategy, tickers, pair) -> pd.DataFrame:
shared_xaxes=True, """
row_width=[1, 1, 4], Get tickers then Populate strategy indicators and signals, then return the full dataframe
vertical_spacing=0.0001, :return: the DataFrame of a pair
) """
fig['layout'].update(title=pair)
fig['layout']['yaxis1'].update(title='Price') dataframes = strategy.tickerdata_to_dataframe(tickers)
fig['layout']['yaxis2'].update(title='Volume') dataframe = dataframes[pair]
fig['layout']['yaxis3'].update(title='Other') dataframe = strategy.advise_buy(dataframe, {'pair': pair})
dataframe = strategy.advise_sell(dataframe, {'pair': pair})
# Common information
candles = go.Candlestick( return dataframe
x=data.date,
open=data.open,
high=data.high, def extract_trades_of_period(dataframe, trades) -> pd.DataFrame:
low=data.low, """
close=data.close, Compare trades and backtested pair DataFrames to get trades performed on backtested period
name='Price' :return: the DataFrame of a trades of period
) """
trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']]
df_buy = data[data['buy'] == 1] return trades
buys = go.Scattergl(
x=df_buy.date,
y=df_buy.close, def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots:
mode='markers', """
name='buy', Generate the graph from the data generated by Backtesting or from DB
marker=dict( :param pair: Pair to Display on the graph
symbol='triangle-up-dot', :param trades: All trades created
size=9, :param data: Dataframe
line=dict(width=1), :param args: sys.argv that contrains the two params indicators1, and indicators2
color='green', :return: None
) """
)
df_sell = data[data['sell'] == 1] # Define the graph
sells = go.Scattergl( fig = tools.make_subplots(
x=df_sell.date, rows=3,
y=df_sell.close, cols=1,
mode='markers', shared_xaxes=True,
name='sell', row_width=[1, 1, 4],
marker=dict( vertical_spacing=0.0001,
symbol='triangle-down-dot', )
size=9, fig['layout'].update(title=pair)
line=dict(width=1), fig['layout']['yaxis1'].update(title='Price')
color='red', fig['layout']['yaxis2'].update(title='Volume')
) fig['layout']['yaxis3'].update(title='Other')
) fig['layout']['xaxis']['rangeslider'].update(visible=False)
trade_buys = go.Scattergl( # Common information
x=trades["opents"], candles = go.Candlestick(
y=trades["open_rate"], x=data.date,
mode='markers', open=data.open,
name='trade_buy', high=data.high,
marker=dict( low=data.low,
symbol='square-open', close=data.close,
size=11, name='Price'
line=dict(width=2), )
color='green'
) df_buy = data[data['buy'] == 1]
) buys = go.Scattergl(
trade_sells = go.Scattergl( x=df_buy.date,
x=trades["closets"], y=df_buy.close,
y=trades["close_rate"], mode='markers',
mode='markers', name='buy',
name='trade_sell', marker=dict(
marker=dict( symbol='triangle-up-dot',
symbol='square-open', size=9,
size=11, line=dict(width=1),
line=dict(width=2), color='green',
color='red' )
) )
) df_sell = data[data['sell'] == 1]
sells = go.Scattergl(
# Row 1 x=df_sell.date,
fig.append_trace(candles, 1, 1) y=df_sell.close,
mode='markers',
if 'bb_lowerband' in data and 'bb_upperband' in data: name='sell',
bb_lower = go.Scatter( marker=dict(
x=data.date, symbol='triangle-down-dot',
y=data.bb_lowerband, size=9,
name='BB lower', line=dict(width=1),
line={'color': 'rgba(255,255,255,0)'}, color='red',
) )
bb_upper = go.Scatter( )
x=data.date,
y=data.bb_upperband, trade_buys = go.Scattergl(
name='BB upper', x=trades["opents"],
fill="tonexty", y=trades["open_rate"],
fillcolor="rgba(0,176,246,0.2)", mode='markers',
line={'color': 'rgba(255,255,255,0)'}, name='trade_buy',
) marker=dict(
fig.append_trace(bb_lower, 1, 1) symbol='square-open',
fig.append_trace(bb_upper, 1, 1) size=11,
line=dict(width=2),
fig = generate_row(fig=fig, row=1, raw_indicators=args.indicators1, data=data) color='green'
fig.append_trace(buys, 1, 1) )
fig.append_trace(sells, 1, 1) )
fig.append_trace(trade_buys, 1, 1) trade_sells = go.Scattergl(
fig.append_trace(trade_sells, 1, 1) x=trades["closets"],
y=trades["close_rate"],
# Row 2 mode='markers',
volume = go.Bar( name='trade_sell',
x=data['date'], marker=dict(
y=data['volume'], symbol='square-open',
name='Volume' size=11,
) line=dict(width=2),
fig.append_trace(volume, 2, 1) color='red'
)
# Row 3 )
fig = generate_row(fig=fig, row=3, raw_indicators=args.indicators2, data=data)
# Row 1
return fig fig.append_trace(candles, 1, 1)
if 'bb_lowerband' in data and 'bb_upperband' in data:
def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots: bb_lower = go.Scatter(
""" x=data.date,
Generator all the indicator selected by the user for a specific row y=data.bb_lowerband,
""" name='BB lower',
for indicator in raw_indicators.split(','): line={'color': 'rgba(255,255,255,0)'},
if indicator in data: )
scattergl = go.Scattergl( bb_upper = go.Scatter(
x=data['date'], x=data.date,
y=data[indicator], y=data.bb_upperband,
name=indicator name='BB upper',
) fill="tonexty",
fig.append_trace(scattergl, row, 1) fillcolor="rgba(0,176,246,0.2)",
else: line={'color': 'rgba(255,255,255,0)'},
logger.info( )
'Indicator "%s" ignored. Reason: This indicator is not found ' fig.append_trace(bb_lower, 1, 1)
'in your strategy.', fig.append_trace(bb_upper, 1, 1)
indicator
) fig = generate_row(fig=fig, row=1, raw_indicators=args.indicators1, data=data)
fig.append_trace(buys, 1, 1)
return fig fig.append_trace(sells, 1, 1)
fig.append_trace(trade_buys, 1, 1)
fig.append_trace(trade_sells, 1, 1)
def plot_parse_args(args: List[str]) -> Namespace:
""" # Row 2
Parse args passed to the script volume = go.Bar(
:param args: Cli arguments x=data['date'],
:return: args: Array with all arguments y=data['volume'],
""" name='Volume'
arguments = Arguments(args, 'Graph dataframe') )
arguments.scripts_options() fig.append_trace(volume, 2, 1)
arguments.parser.add_argument(
'--indicators1', # Row 3
help='Set indicators from your strategy you want in the first row of the graph. Separate ' fig = generate_row(fig=fig, row=3, raw_indicators=args.indicators2, data=data)
'them with a coma. E.g: ema3,ema5 (default: %(default)s)',
type=str, return fig
default='sma,ema3,ema5',
dest='indicators1',
) def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots:
"""
arguments.parser.add_argument( Generator all the indicator selected by the user for a specific row
'--indicators2', """
help='Set indicators from your strategy you want in the third row of the graph. Separate ' for indicator in raw_indicators.split(','):
'them with a coma. E.g: fastd,fastk (default: %(default)s)', if indicator in data:
type=str, scattergl = go.Scattergl(
default='macd', x=data['date'],
dest='indicators2', y=data[indicator],
) name=indicator
arguments.parser.add_argument( )
'--plot-limit', fig.append_trace(scattergl, row, 1)
help='Specify tick limit for plotting - too high values cause huge files - ' else:
'Default: %(default)s', logger.info(
dest='plot_limit', 'Indicator "%s" ignored. Reason: This indicator is not found '
default=750, 'in your strategy.',
type=int, indicator
) )
arguments.common_args_parser()
arguments.optimizer_shared_options(arguments.parser) return fig
arguments.backtesting_options(arguments.parser)
return arguments.parse_args()
def plot_parse_args(args: List[str]) -> Namespace:
"""
def main(sysargv: List[str]) -> None: Parse args passed to the script
""" :param args: Cli arguments
This function will initiate the bot and start the trading loop. :return: args: Array with all arguments
:return: None """
""" arguments = Arguments(args, 'Graph dataframe')
logger.info('Starting Plot Dataframe') arguments.scripts_options()
plot_analyzed_dataframe( arguments.parser.add_argument(
plot_parse_args(sysargv) '--indicators1',
) help='Set indicators from your strategy you want in the first row of the graph. Separate '
'them with a coma. E.g: ema3,ema5 (default: %(default)s)',
type=str,
if __name__ == '__main__': default='sma,ema3,ema5',
main(sys.argv[1:]) dest='indicators1',
)
arguments.parser.add_argument(
'--indicators2',
help='Set indicators from your strategy you want in the third row of the graph. Separate '
'them with a coma. E.g: fastd,fastk (default: %(default)s)',
type=str,
default='macd',
dest='indicators2',
)
arguments.parser.add_argument(
'--plot-limit',
help='Specify tick limit for plotting - too high values cause huge files - '
'Default: %(default)s',
dest='plot_limit',
default=750,
type=int,
)
arguments.common_args_parser()
arguments.optimizer_shared_options(arguments.parser)
arguments.backtesting_options(arguments.parser)
return arguments.parse_args()
def analyse_and_plot_pairs(args: Namespace):
"""
From arguments provided in cli:
-Initialise backtest env
-Get tickers data
-Generate Dafaframes populated with indicators and signals
-Load trades excecuted on same periods
-Generate Plotly plot objects
-Generate plot files
:return: None
"""
strategy, exchange, pairs = get_trading_env(args)
# Set timerange to use
timerange = Arguments.parse_timerange(args.timerange)
tick_interval = strategy.ticker_interval
tickers = get_tickers_data(strategy, exchange, pairs, args)
pair_counter = 0
for pair, data in tickers.items():
pair_counter += 1
logger.info("analyse pair %s", pair)
tickers = {}
tickers[pair] = data
dataframe = generate_dataframe(strategy, tickers, pair)
trades = load_trades(args, pair, timerange)
trades = extract_trades_of_period(dataframe, trades)
fig = generate_graph(
pair=pair,
trades=trades,
data=dataframe,
args=args
)
is_last = (False, True)[pair_counter == len(tickers)]
generate_plot_file(fig, pair, tick_interval, is_last)
logger.info('End of ploting process %s plots generated', pair_counter)
def main(sysargv: List[str]) -> None:
"""
This function will initiate the bot and start the trading loop.
:return: None
"""
logger.info('Starting Plot Dataframe')
analyse_and_plot_pairs(
plot_parse_args(sysargv)
)
exit()
if __name__ == '__main__':
main(sys.argv[1:])

4
scripts/plot_profit.py
View File

@ -107,8 +107,8 @@ def plot_profit(args: Namespace) -> None:
exit(1) exit(1)
# Take pairs from the cli otherwise switch to the pair in the config file # Take pairs from the cli otherwise switch to the pair in the config file
if args.pair: if args.pairs:
filter_pairs = args.pair filter_pairs = args.pairs
filter_pairs = filter_pairs.split(',') filter_pairs = filter_pairs.split(',')
else: else:
filter_pairs = config['exchange']['pair_whitelist'] filter_pairs = config['exchange']['pair_whitelist']