Change back to LF lineendings

This commit is contained in:
Matthias 2019-01-25 06:42:29 +01:00
parent b840b9f53a
commit 22e7ad8ec1

View File

@ -1,460 +1,460 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
Script to display when the bot will buy on specific pair(s) Script to display when the bot will buy on specific pair(s)
Mandatory Cli parameters: Mandatory Cli parameters:
-p / --pairs: pair(s) to examine -p / --pairs: pair(s) to examine
Option but recommended Option but recommended
-s / --strategy: strategy to use -s / --strategy: strategy to use
Optional Cli parameters Optional Cli parameters
-d / --datadir: path to pair(s) backtest data -d / --datadir: path to pair(s) backtest data
--timerange: specify what timerange of data to use. --timerange: specify what timerange of data to use.
-l / --live: Live, to download the latest ticker for the pair(s) -l / --live: Live, to download the latest ticker for the pair(s)
-db / --db-url: Show trades stored in database -db / --db-url: Show trades stored in database
Indicators recommended Indicators recommended
Row 1: sma, ema3, ema5, ema10, ema50 Row 1: sma, ema3, ema5, ema10, ema50
Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk
Example of usage: Example of usage:
> python3 scripts/plot_dataframe.py --pairs BTC/EUR,XRP/BTC -d user_data/data/ --indicators1 sma,ema3 > python3 scripts/plot_dataframe.py --pairs BTC/EUR,XRP/BTC -d user_data/data/ --indicators1 sma,ema3
--indicators2 fastk,fastd --indicators2 fastk,fastd
""" """
import json import json
import logging import logging
import sys import sys
import os import os
from argparse import Namespace from argparse import Namespace
from pathlib import Path from pathlib import Path
from typing import Dict, List, Any from typing import Dict, List, Any
import pandas as pd import pandas as pd
import plotly.graph_objs as go import plotly.graph_objs as go
import pytz import pytz
from plotly import tools from plotly import tools
from plotly.offline import plot from plotly.offline import plot
from freqtrade import persistence from freqtrade import persistence
from freqtrade.arguments import Arguments, TimeRange from freqtrade.arguments import Arguments, TimeRange
from freqtrade.data import history from freqtrade.data import history
from freqtrade.exchange import Exchange from freqtrade.exchange import Exchange
from freqtrade.optimize.backtesting import setup_configuration from freqtrade.optimize.backtesting import setup_configuration
from freqtrade.persistence import Trade from freqtrade.persistence import Trade
from freqtrade.resolvers import StrategyResolver from freqtrade.resolvers import StrategyResolver
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_CONF: Dict[str, Any] = {} _CONF: Dict[str, Any] = {}
timeZone = pytz.UTC timeZone = pytz.UTC
def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame: def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame:
trades: pd.DataFrame = pd.DataFrame() trades: pd.DataFrame = pd.DataFrame()
if args.db_url: if args.db_url:
persistence.init(_CONF) persistence.init(_CONF)
columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"] columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"]
for x in Trade.query.all(): for x in Trade.query.all():
print("date: {}".format(x.open_date)) print("date: {}".format(x.open_date))
trades = pd.DataFrame([(t.pair, t.calc_profit(), trades = pd.DataFrame([(t.pair, t.calc_profit(),
t.open_date.replace(tzinfo=timeZone), t.open_date.replace(tzinfo=timeZone),
t.close_date.replace(tzinfo=timeZone) if t.close_date else None, t.close_date.replace(tzinfo=timeZone) if t.close_date else None,
t.open_rate, t.close_rate, t.open_rate, t.close_rate,
t.close_date.timestamp() - t.open_date.timestamp() t.close_date.timestamp() - t.open_date.timestamp()
if t.close_date else None) if t.close_date else None)
for t in Trade.query.filter(Trade.pair.is_(pair)).all()], for t in Trade.query.filter(Trade.pair.is_(pair)).all()],
columns=columns) columns=columns)
elif args.exportfilename: elif args.exportfilename:
file = Path(args.exportfilename) file = Path(args.exportfilename)
# must align with columns in backtest.py # must align with columns in backtest.py
columns = ["pair", "profit", "opents", "closets", "index", "duration", columns = ["pair", "profit", "opents", "closets", "index", "duration",
"open_rate", "close_rate", "open_at_end", "sell_reason"] "open_rate", "close_rate", "open_at_end", "sell_reason"]
if os.path.exists(file): if os.path.exists(file):
with file.open() as f: with file.open() as f:
data = json.load(f) data = json.load(f)
trades = pd.DataFrame(data, columns=columns) trades = pd.DataFrame(data, columns=columns)
trades = trades.loc[trades["pair"] == pair] trades = trades.loc[trades["pair"] == pair]
if timerange: if timerange:
if timerange.starttype == 'date': if timerange.starttype == 'date':
trades = trades.loc[trades["opents"] >= timerange.startts] trades = trades.loc[trades["opents"] >= timerange.startts]
if timerange.stoptype == 'date': if timerange.stoptype == 'date':
trades = trades.loc[trades["opents"] <= timerange.stopts] trades = trades.loc[trades["opents"] <= timerange.stopts]
trades['opents'] = pd.to_datetime( trades['opents'] = pd.to_datetime(
trades['opents'], trades['opents'],
unit='s', unit='s',
utc=True, utc=True,
infer_datetime_format=True) infer_datetime_format=True)
trades['closets'] = pd.to_datetime( trades['closets'] = pd.to_datetime(
trades['closets'], trades['closets'],
unit='s', unit='s',
utc=True, utc=True,
infer_datetime_format=True) infer_datetime_format=True)
else: else:
trades = pd.DataFrame([], columns=columns) trades = pd.DataFrame([], columns=columns)
return trades return trades
def generate_plot_file(fig, pair, tick_interval, is_last) -> None: def generate_plot_file(fig, pair, tick_interval, is_last) -> None:
""" """
Generate a plot html file from pre populated fig plotly object Generate a plot html file from pre populated fig plotly object
:return: None :return: None
""" """
logger.info('Generate plot file for %s', pair) logger.info('Generate plot file for %s', pair)
pair_name = pair.replace("/", "_") pair_name = pair.replace("/", "_")
file_name = 'freqtrade-plot-' + pair_name + '-' + tick_interval + '.html' file_name = 'freqtrade-plot-' + pair_name + '-' + tick_interval + '.html'
if not os.path.exists('user_data/plots'): if not os.path.exists('user_data/plots'):
try: try:
os.makedirs('user_data/plots') os.makedirs('user_data/plots')
except OSError as e: except OSError as e:
raise raise
plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), auto_open=False) plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), auto_open=False)
if is_last: if is_last:
plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')), auto_open=False) plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')), auto_open=False)
def get_trading_env(args: Namespace): def get_trading_env(args: Namespace):
""" """
Initalize freqtrade Exchange and Strategy, split pairs recieved in parameter Initalize freqtrade Exchange and Strategy, split pairs recieved in parameter
:return: Strategy :return: Strategy
""" """
global _CONF global _CONF
# Load the configuration # Load the configuration
_CONF.update(setup_configuration(args)) _CONF.update(setup_configuration(args))
print(_CONF) print(_CONF)
pairs = args.pairs.split(',') pairs = args.pairs.split(',')
if pairs is None: if pairs is None:
logger.critical('Parameter --pairs mandatory;. E.g --pairs ETH/BTC,XRP/BTC') logger.critical('Parameter --pairs mandatory;. E.g --pairs ETH/BTC,XRP/BTC')
exit() exit()
# Load the strategy # Load the strategy
try: try:
strategy = StrategyResolver(_CONF).strategy strategy = StrategyResolver(_CONF).strategy
exchange = Exchange(_CONF) exchange = Exchange(_CONF)
except AttributeError: except AttributeError:
logger.critical( logger.critical(
'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"',
args.strategy args.strategy
) )
exit() exit()
return [strategy, exchange, pairs] return [strategy, exchange, pairs]
def get_tickers_data(strategy, exchange, pairs: List[str], args): def get_tickers_data(strategy, exchange, pairs: List[str], args):
""" """
Get tickers data for each pairs on live or local, option defined in args Get tickers data for each pairs on live or local, option defined in args
:return: dictinnary of tickers. output format: {'pair': tickersdata} :return: dictinnary of tickers. output format: {'pair': tickersdata}
""" """
tick_interval = strategy.ticker_interval tick_interval = strategy.ticker_interval
timerange = Arguments.parse_timerange(args.timerange) timerange = Arguments.parse_timerange(args.timerange)
tickers = {} tickers = {}
if args.live: if args.live:
logger.info('Downloading pairs.') logger.info('Downloading pairs.')
exchange.refresh_tickers(pairs, tick_interval) exchange.refresh_tickers(pairs, tick_interval)
for pair in pairs: for pair in pairs:
tickers[pair] = exchange.klines(pair) tickers[pair] = exchange.klines(pair)
else: else:
tickers = history.load_data( tickers = history.load_data(
datadir=Path(_CONF.get("datadir")), datadir=Path(_CONF.get("datadir")),
pairs=pairs, pairs=pairs,
ticker_interval=tick_interval, ticker_interval=tick_interval,
refresh_pairs=_CONF.get('refresh_pairs', False), refresh_pairs=_CONF.get('refresh_pairs', False),
timerange=timerange, timerange=timerange,
exchange=Exchange(_CONF) exchange=Exchange(_CONF)
) )
# No ticker found, impossible to download, len mismatch # No ticker found, impossible to download, len mismatch
for pair, data in tickers.copy().items(): for pair, data in tickers.copy().items():
logger.debug("checking tickers data of pair: %s", pair) logger.debug("checking tickers data of pair: %s", pair)
logger.debug("data.empty: %s", data.empty) logger.debug("data.empty: %s", data.empty)
logger.debug("len(data): %s", len(data)) logger.debug("len(data): %s", len(data))
if data.empty: if data.empty:
del tickers[pair] del tickers[pair]
logger.info( logger.info(
'An issue occured while retreiving datas of %s pair, please retry ' 'An issue occured while retreiving datas of %s pair, please retry '
'using -l option for live or --refresh-pairs-cached', pair) 'using -l option for live or --refresh-pairs-cached', pair)
return tickers return tickers
def generate_dataframe(strategy, tickers, pair) -> pd.DataFrame: def generate_dataframe(strategy, tickers, pair) -> pd.DataFrame:
""" """
Get tickers then Populate strategy indicators and signals, then return the full dataframe Get tickers then Populate strategy indicators and signals, then return the full dataframe
:return: the DataFrame of a pair :return: the DataFrame of a pair
""" """
dataframes = strategy.tickerdata_to_dataframe(tickers) dataframes = strategy.tickerdata_to_dataframe(tickers)
dataframe = dataframes[pair] dataframe = dataframes[pair]
dataframe = strategy.advise_buy(dataframe, {'pair': pair}) dataframe = strategy.advise_buy(dataframe, {'pair': pair})
dataframe = strategy.advise_sell(dataframe, {'pair': pair}) dataframe = strategy.advise_sell(dataframe, {'pair': pair})
return dataframe return dataframe
def extract_trades_of_period(dataframe, trades) -> pd.DataFrame: def extract_trades_of_period(dataframe, trades) -> pd.DataFrame:
""" """
Compare trades and backtested pair DataFrames to get trades performed on backtested period Compare trades and backtested pair DataFrames to get trades performed on backtested period
:return: the DataFrame of a trades of period :return: the DataFrame of a trades of period
""" """
trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']] trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']]
return trades return trades
def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots: def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots:
""" """
Generate the graph from the data generated by Backtesting or from DB Generate the graph from the data generated by Backtesting or from DB
:param pair: Pair to Display on the graph :param pair: Pair to Display on the graph
:param trades: All trades created :param trades: All trades created
:param data: Dataframe :param data: Dataframe
:param args: sys.argv that contrains the two params indicators1, and indicators2 :param args: sys.argv that contrains the two params indicators1, and indicators2
:return: None :return: None
""" """
# Define the graph # Define the graph
fig = tools.make_subplots( fig = tools.make_subplots(
rows=3, rows=3,
cols=1, cols=1,
shared_xaxes=True, shared_xaxes=True,
row_width=[1, 1, 4], row_width=[1, 1, 4],
vertical_spacing=0.0001, vertical_spacing=0.0001,
) )
fig['layout'].update(title=pair) fig['layout'].update(title=pair)
fig['layout']['yaxis1'].update(title='Price') fig['layout']['yaxis1'].update(title='Price')
fig['layout']['yaxis2'].update(title='Volume') fig['layout']['yaxis2'].update(title='Volume')
fig['layout']['yaxis3'].update(title='Other') fig['layout']['yaxis3'].update(title='Other')
fig['layout']['xaxis']['rangeslider'].update(visible=False) fig['layout']['xaxis']['rangeslider'].update(visible=False)
# Common information # Common information
candles = go.Candlestick( candles = go.Candlestick(
x=data.date, x=data.date,
open=data.open, open=data.open,
high=data.high, high=data.high,
low=data.low, low=data.low,
close=data.close, close=data.close,
name='Price' name='Price'
) )
df_buy = data[data['buy'] == 1] df_buy = data[data['buy'] == 1]
buys = go.Scattergl( buys = go.Scattergl(
x=df_buy.date, x=df_buy.date,
y=df_buy.close, y=df_buy.close,
mode='markers', mode='markers',
name='buy', name='buy',
marker=dict( marker=dict(
symbol='triangle-up-dot', symbol='triangle-up-dot',
size=9, size=9,
line=dict(width=1), line=dict(width=1),
color='green', color='green',
) )
) )
df_sell = data[data['sell'] == 1] df_sell = data[data['sell'] == 1]
sells = go.Scattergl( sells = go.Scattergl(
x=df_sell.date, x=df_sell.date,
y=df_sell.close, y=df_sell.close,
mode='markers', mode='markers',
name='sell', name='sell',
marker=dict( marker=dict(
symbol='triangle-down-dot', symbol='triangle-down-dot',
size=9, size=9,
line=dict(width=1), line=dict(width=1),
color='red', color='red',
) )
) )
trade_buys = go.Scattergl( trade_buys = go.Scattergl(
x=trades["opents"], x=trades["opents"],
y=trades["open_rate"], y=trades["open_rate"],
mode='markers', mode='markers',
name='trade_buy', name='trade_buy',
marker=dict( marker=dict(
symbol='square-open', symbol='square-open',
size=11, size=11,
line=dict(width=2), line=dict(width=2),
color='green' color='green'
) )
) )
trade_sells = go.Scattergl( trade_sells = go.Scattergl(
x=trades["closets"], x=trades["closets"],
y=trades["close_rate"], y=trades["close_rate"],
mode='markers', mode='markers',
name='trade_sell', name='trade_sell',
marker=dict( marker=dict(
symbol='square-open', symbol='square-open',
size=11, size=11,
line=dict(width=2), line=dict(width=2),
color='red' color='red'
) )
) )
# Row 1 # Row 1
fig.append_trace(candles, 1, 1) fig.append_trace(candles, 1, 1)
if 'bb_lowerband' in data and 'bb_upperband' in data: if 'bb_lowerband' in data and 'bb_upperband' in data:
bb_lower = go.Scatter( bb_lower = go.Scatter(
x=data.date, x=data.date,
y=data.bb_lowerband, y=data.bb_lowerband,
name='BB lower', name='BB lower',
line={'color': 'rgba(255,255,255,0)'}, line={'color': 'rgba(255,255,255,0)'},
) )
bb_upper = go.Scatter( bb_upper = go.Scatter(
x=data.date, x=data.date,
y=data.bb_upperband, y=data.bb_upperband,
name='BB upper', name='BB upper',
fill="tonexty", fill="tonexty",
fillcolor="rgba(0,176,246,0.2)", fillcolor="rgba(0,176,246,0.2)",
line={'color': 'rgba(255,255,255,0)'}, line={'color': 'rgba(255,255,255,0)'},
) )
fig.append_trace(bb_lower, 1, 1) fig.append_trace(bb_lower, 1, 1)
fig.append_trace(bb_upper, 1, 1) fig.append_trace(bb_upper, 1, 1)
fig = generate_row(fig=fig, row=1, raw_indicators=args.indicators1, data=data) fig = generate_row(fig=fig, row=1, raw_indicators=args.indicators1, data=data)
fig.append_trace(buys, 1, 1) fig.append_trace(buys, 1, 1)
fig.append_trace(sells, 1, 1) fig.append_trace(sells, 1, 1)
fig.append_trace(trade_buys, 1, 1) fig.append_trace(trade_buys, 1, 1)
fig.append_trace(trade_sells, 1, 1) fig.append_trace(trade_sells, 1, 1)
# Row 2 # Row 2
volume = go.Bar( volume = go.Bar(
x=data['date'], x=data['date'],
y=data['volume'], y=data['volume'],
name='Volume' name='Volume'
) )
fig.append_trace(volume, 2, 1) fig.append_trace(volume, 2, 1)
# Row 3 # Row 3
fig = generate_row(fig=fig, row=3, raw_indicators=args.indicators2, data=data) fig = generate_row(fig=fig, row=3, raw_indicators=args.indicators2, data=data)
return fig return fig
def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots: def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots:
""" """
Generator all the indicator selected by the user for a specific row Generator all the indicator selected by the user for a specific row
""" """
for indicator in raw_indicators.split(','): for indicator in raw_indicators.split(','):
if indicator in data: if indicator in data:
scattergl = go.Scattergl( scattergl = go.Scattergl(
x=data['date'], x=data['date'],
y=data[indicator], y=data[indicator],
name=indicator name=indicator
) )
fig.append_trace(scattergl, row, 1) fig.append_trace(scattergl, row, 1)
else: else:
logger.info( logger.info(
'Indicator "%s" ignored. Reason: This indicator is not found ' 'Indicator "%s" ignored. Reason: This indicator is not found '
'in your strategy.', 'in your strategy.',
indicator indicator
) )
return fig return fig
def plot_parse_args(args: List[str]) -> Namespace: def plot_parse_args(args: List[str]) -> Namespace:
""" """
Parse args passed to the script Parse args passed to the script
:param args: Cli arguments :param args: Cli arguments
:return: args: Array with all arguments :return: args: Array with all arguments
""" """
arguments = Arguments(args, 'Graph dataframe') arguments = Arguments(args, 'Graph dataframe')
arguments.scripts_options() arguments.scripts_options()
arguments.parser.add_argument( arguments.parser.add_argument(
'--indicators1', '--indicators1',
help='Set indicators from your strategy you want in the first row of the graph. Separate ' help='Set indicators from your strategy you want in the first row of the graph. Separate '
'them with a coma. E.g: ema3,ema5 (default: %(default)s)', 'them with a coma. E.g: ema3,ema5 (default: %(default)s)',
type=str, type=str,
default='sma,ema3,ema5', default='sma,ema3,ema5',
dest='indicators1', dest='indicators1',
) )
arguments.parser.add_argument( arguments.parser.add_argument(
'--indicators2', '--indicators2',
help='Set indicators from your strategy you want in the third row of the graph. Separate ' help='Set indicators from your strategy you want in the third row of the graph. Separate '
'them with a coma. E.g: fastd,fastk (default: %(default)s)', 'them with a coma. E.g: fastd,fastk (default: %(default)s)',
type=str, type=str,
default='macd', default='macd',
dest='indicators2', dest='indicators2',
) )
arguments.parser.add_argument( arguments.parser.add_argument(
'--plot-limit', '--plot-limit',
help='Specify tick limit for plotting - too high values cause huge files - ' help='Specify tick limit for plotting - too high values cause huge files - '
'Default: %(default)s', 'Default: %(default)s',
dest='plot_limit', dest='plot_limit',
default=750, default=750,
type=int, type=int,
) )
arguments.common_args_parser() arguments.common_args_parser()
arguments.optimizer_shared_options(arguments.parser) arguments.optimizer_shared_options(arguments.parser)
arguments.backtesting_options(arguments.parser) arguments.backtesting_options(arguments.parser)
return arguments.parse_args() return arguments.parse_args()
def analyse_and_plot_pairs(args: Namespace): def analyse_and_plot_pairs(args: Namespace):
""" """
From arguments provided in cli: From arguments provided in cli:
-Initialise backtest env -Initialise backtest env
-Get tickers data -Get tickers data
-Generate Dafaframes populated with indicators and signals -Generate Dafaframes populated with indicators and signals
-Load trades excecuted on same periods -Load trades excecuted on same periods
-Generate Plotly plot objects -Generate Plotly plot objects
-Generate plot files -Generate plot files
:return: None :return: None
""" """
strategy, exchange, pairs = get_trading_env(args) strategy, exchange, pairs = get_trading_env(args)
# Set timerange to use # Set timerange to use
timerange = Arguments.parse_timerange(args.timerange) timerange = Arguments.parse_timerange(args.timerange)
tick_interval = strategy.ticker_interval tick_interval = strategy.ticker_interval
tickers = get_tickers_data(strategy, exchange, pairs, args) tickers = get_tickers_data(strategy, exchange, pairs, args)
pair_counter = 0 pair_counter = 0
for pair, data in tickers.items(): for pair, data in tickers.items():
pair_counter += 1 pair_counter += 1
logger.info("analyse pair %s", pair) logger.info("analyse pair %s", pair)
tickers = {} tickers = {}
tickers[pair] = data tickers[pair] = data
dataframe = generate_dataframe(strategy, tickers, pair) dataframe = generate_dataframe(strategy, tickers, pair)
trades = load_trades(args, pair, timerange) trades = load_trades(args, pair, timerange)
trades = extract_trades_of_period(dataframe, trades) trades = extract_trades_of_period(dataframe, trades)
fig = generate_graph( fig = generate_graph(
pair=pair, pair=pair,
trades=trades, trades=trades,
data=dataframe, data=dataframe,
args=args args=args
) )
is_last = (False, True)[pair_counter == len(tickers)] is_last = (False, True)[pair_counter == len(tickers)]
generate_plot_file(fig, pair, tick_interval, is_last) generate_plot_file(fig, pair, tick_interval, is_last)
logger.info('End of ploting process %s plots generated', pair_counter) logger.info('End of ploting process %s plots generated', pair_counter)
def main(sysargv: List[str]) -> None: def main(sysargv: List[str]) -> None:
""" """
This function will initiate the bot and start the trading loop. This function will initiate the bot and start the trading loop.
:return: None :return: None
""" """
logger.info('Starting Plot Dataframe') logger.info('Starting Plot Dataframe')
analyse_and_plot_pairs( analyse_and_plot_pairs(
plot_parse_args(sysargv) plot_parse_args(sysargv)
) )
exit() exit()
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[1:]) main(sys.argv[1:])