Merge pull request #1287 from freqtrade/backtest_data_validation

Backtest data validation
This commit is contained in:
Matthias 2018-10-26 07:21:24 -04:00 committed by GitHub
commit 7e1a30f9bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 124 additions and 437 deletions

View File

@ -10,8 +10,12 @@ except ImportError:
_UJSON = False _UJSON = False
import logging import logging
import os import os
from datetime import datetime
from typing import Optional, List, Dict, Tuple, Any from typing import Optional, List, Dict, Tuple, Any
import operator
import arrow import arrow
from pandas import DataFrame
from freqtrade import misc, constants, OperationalException from freqtrade import misc, constants, OperationalException
from freqtrade.exchange import Exchange from freqtrade.exchange import Exchange
@ -59,6 +63,42 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
return tickerlist[start_index:stop_index] return tickerlist[start_index:stop_index]
def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
"""
Get the maximum timeframe for the given backtest data
:param data: dictionary with preprocessed backtesting data
:return: tuple containing min_date, max_date
"""
timeframe = [
(arrow.get(frame['date'].min()), arrow.get(frame['date'].max()))
for frame in data.values()
]
return min(timeframe, key=operator.itemgetter(0))[0], \
max(timeframe, key=operator.itemgetter(1))[1]
def validate_backtest_data(data: Dict[str, DataFrame], min_date: datetime,
max_date: datetime, ticker_interval_mins: int) -> bool:
"""
Validates preprocessed backtesting data for missing values and shows warnings about it that.
:param data: dictionary with preprocessed backtesting data
:param min_date: start-date of the data
:param max_date: end-date of the data
:param ticker_interval_mins: ticker interval in minutes
"""
# total difference in minutes / interval-minutes
expected_frames = int((max_date - min_date).total_seconds() // 60 // ticker_interval_mins)
found_missing = False
for pair, df in data.items():
dflen = len(df)
if dflen < expected_frames:
found_missing = True
logger.warning("%s has missing frames: expected %s, got %s, that's %s missing values",
pair, expected_frames, dflen, expected_frames - dflen)
return found_missing
def load_tickerdata_file( def load_tickerdata_file(
datadir: str, pair: str, datadir: str, pair: str,
ticker_interval: str, ticker_interval: str,

View File

@ -4,14 +4,12 @@
This module contains the backtesting logic This module contains the backtesting logic
""" """
import logging import logging
import operator
from argparse import Namespace from argparse import Namespace
from copy import deepcopy from copy import deepcopy
from datetime import datetime, timedelta from datetime import datetime, timedelta
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, NamedTuple, Optional, Tuple from typing import Any, Dict, List, NamedTuple, Optional
import arrow
from pandas import DataFrame from pandas import DataFrame
from tabulate import tabulate from tabulate import tabulate
@ -88,24 +86,9 @@ class Backtesting(object):
""" """
self.strategy = strategy self.strategy = strategy
self.ticker_interval = self.config.get('ticker_interval') self.ticker_interval = self.config.get('ticker_interval')
self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe
self.advise_buy = strategy.advise_buy self.advise_buy = strategy.advise_buy
self.advise_sell = strategy.advise_sell self.advise_sell = strategy.advise_sell
@staticmethod
def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
"""
Get the maximum timeframe for the given backtest data
:param data: dictionary with preprocessed backtesting data
:return: tuple containing min_date, max_date
"""
timeframe = [
(arrow.get(frame['date'].min()), arrow.get(frame['date'].max()))
for frame in data.values()
]
return min(timeframe, key=operator.itemgetter(0))[0], \
max(timeframe, key=operator.itemgetter(1))[1]
def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame, def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame,
skip_nan: bool = False) -> str: skip_nan: bool = False) -> str:
""" """
@ -371,10 +354,12 @@ class Backtesting(object):
self._set_strategy(strat) self._set_strategy(strat)
# need to reprocess data every time to populate signals # need to reprocess data every time to populate signals
preprocessed = self.tickerdata_to_dataframe(data) preprocessed = self.strategy.tickerdata_to_dataframe(data)
# Print timeframe min_date, max_date = optimize.get_timeframe(preprocessed)
min_date, max_date = self.get_timeframe(preprocessed) # Validate dataframe for missing values
optimize.validate_backtest_data(preprocessed, min_date, max_date,
constants.TICKER_INTERVAL_MINUTES[self.ticker_interval])
logger.info( logger.info(
'Measuring data from %s up to %s (%s days)..', 'Measuring data from %s up to %s (%s days)..',
min_date.isoformat(), min_date.isoformat(),

View File

@ -352,7 +352,7 @@ class Hyperopt(Backtesting):
if self.has_space('buy'): if self.has_space('buy'):
self.strategy.advise_indicators = Hyperopt.populate_indicators # type: ignore self.strategy.advise_indicators = Hyperopt.populate_indicators # type: ignore
dump(self.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE)
self.exchange = None # type: ignore self.exchange = None # type: ignore
self.load_previous_results() self.load_previous_results()

View File

@ -89,7 +89,7 @@ def simple_backtest(config, contour, num_results, mocker) -> None:
backtesting = Backtesting(config) backtesting = Backtesting(config)
data = load_data_test(contour) data = load_data_test(contour)
processed = backtesting.tickerdata_to_dataframe(data) processed = backtesting.strategy.tickerdata_to_dataframe(data)
assert isinstance(processed, dict) assert isinstance(processed, dict)
results = backtesting.backtest( results = backtesting.backtest(
{ {
@ -119,13 +119,13 @@ def _load_pair_as_ticks(pair, tickfreq):
# FIX: fixturize this? # FIX: fixturize this?
def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None): def _make_backtest_conf(mocker, conf=None, pair='UNITTEST/BTC', record=None):
data = optimize.load_data(None, ticker_interval='8m', pairs=[pair]) data = optimize.load_data(None, ticker_interval='1m', pairs=[pair])
data = trim_dictlist(data, -201) data = trim_dictlist(data, -201)
patch_exchange(mocker) patch_exchange(mocker)
backtesting = Backtesting(conf) backtesting = Backtesting(conf)
return { return {
'stake_amount': conf['stake_amount'], 'stake_amount': conf['stake_amount'],
'processed': backtesting.tickerdata_to_dataframe(data), 'processed': backtesting.strategy.tickerdata_to_dataframe(data),
'max_open_trades': 10, 'max_open_trades': 10,
'position_stacking': False, 'position_stacking': False,
'record': record 'record': record
@ -313,7 +313,7 @@ def test_backtesting_init(mocker, default_conf) -> None:
backtesting = Backtesting(default_conf) backtesting = Backtesting(default_conf)
assert backtesting.config == default_conf assert backtesting.config == default_conf
assert backtesting.ticker_interval == '5m' assert backtesting.ticker_interval == '5m'
assert callable(backtesting.tickerdata_to_dataframe) assert callable(backtesting.strategy.tickerdata_to_dataframe)
assert callable(backtesting.advise_buy) assert callable(backtesting.advise_buy)
assert callable(backtesting.advise_sell) assert callable(backtesting.advise_sell)
get_fee.assert_called() get_fee.assert_called()
@ -327,7 +327,7 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None:
tickerlist = {'UNITTEST/BTC': tick} tickerlist = {'UNITTEST/BTC': tick}
backtesting = Backtesting(default_conf) backtesting = Backtesting(default_conf)
data = backtesting.tickerdata_to_dataframe(tickerlist) data = backtesting.strategy.tickerdata_to_dataframe(tickerlist)
assert len(data['UNITTEST/BTC']) == 99 assert len(data['UNITTEST/BTC']) == 99
# Load strategy to compare the result between Backtesting function and strategy are the same # Load strategy to compare the result between Backtesting function and strategy are the same
@ -336,22 +336,6 @@ def test_tickerdata_to_dataframe(default_conf, mocker) -> None:
assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC']) assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC'])
def test_get_timeframe(default_conf, mocker) -> None:
patch_exchange(mocker)
backtesting = Backtesting(default_conf)
data = backtesting.tickerdata_to_dataframe(
optimize.load_data(
None,
ticker_interval='1m',
pairs=['UNITTEST/BTC']
)
)
min_date, max_date = backtesting.get_timeframe(data)
assert min_date.isoformat() == '2017-11-04T23:02:00+00:00'
assert max_date.isoformat() == '2017-11-14T22:58:00+00:00'
def test_generate_text_table(default_conf, mocker): def test_generate_text_table(default_conf, mocker):
patch_exchange(mocker) patch_exchange(mocker)
backtesting = Backtesting(default_conf) backtesting = Backtesting(default_conf)
@ -451,21 +435,21 @@ def test_generate_text_table_strategyn(default_conf, mocker):
def test_backtesting_start(default_conf, mocker, caplog) -> None: def test_backtesting_start(default_conf, mocker, caplog) -> None:
def get_timeframe(input1, input2): def get_timeframe(input1):
return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
mocker.patch('freqtrade.optimize.load_data', mocked_load_data) mocker.patch('freqtrade.optimize.load_data', mocked_load_data)
mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe)
mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock())
patch_exchange(mocker) patch_exchange(mocker)
mocker.patch.multiple( mocker.patch.multiple(
'freqtrade.optimize.backtesting.Backtesting', 'freqtrade.optimize.backtesting.Backtesting',
backtest=MagicMock(), backtest=MagicMock(),
_generate_text_table=MagicMock(return_value='1'), _generate_text_table=MagicMock(return_value='1'),
get_timeframe=get_timeframe,
) )
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
default_conf['ticker_interval'] = 1 default_conf['ticker_interval'] = "1m"
default_conf['live'] = False default_conf['live'] = False
default_conf['datadir'] = None default_conf['datadir'] = None
default_conf['export'] = None default_conf['export'] = None
@ -486,17 +470,17 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None:
def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None: def test_backtesting_start_no_data(default_conf, mocker, caplog) -> None:
def get_timeframe(input1, input2): def get_timeframe(input1):
return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={})) mocker.patch('freqtrade.optimize.load_data', MagicMock(return_value={}))
mocker.patch('freqtrade.optimize.get_timeframe', get_timeframe)
mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock())
patch_exchange(mocker) patch_exchange(mocker)
mocker.patch.multiple( mocker.patch.multiple(
'freqtrade.optimize.backtesting.Backtesting', 'freqtrade.optimize.backtesting.Backtesting',
backtest=MagicMock(), backtest=MagicMock(),
_generate_text_table=MagicMock(return_value='1'), _generate_text_table=MagicMock(return_value='1'),
get_timeframe=get_timeframe,
) )
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
@ -520,7 +504,7 @@ def test_backtest(default_conf, fee, mocker) -> None:
pair = 'UNITTEST/BTC' pair = 'UNITTEST/BTC'
data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC']) data = optimize.load_data(None, ticker_interval='5m', pairs=['UNITTEST/BTC'])
data = trim_dictlist(data, -200) data = trim_dictlist(data, -200)
data_processed = backtesting.tickerdata_to_dataframe(data) data_processed = backtesting.strategy.tickerdata_to_dataframe(data)
results = backtesting.backtest( results = backtesting.backtest(
{ {
'stake_amount': default_conf['stake_amount'], 'stake_amount': default_conf['stake_amount'],
@ -571,7 +555,7 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None:
results = backtesting.backtest( results = backtesting.backtest(
{ {
'stake_amount': default_conf['stake_amount'], 'stake_amount': default_conf['stake_amount'],
'processed': backtesting.tickerdata_to_dataframe(data), 'processed': backtesting.strategy.tickerdata_to_dataframe(data),
'max_open_trades': 1, 'max_open_trades': 1,
'position_stacking': False 'position_stacking': False
} }
@ -585,7 +569,7 @@ def test_processed(default_conf, mocker) -> None:
backtesting = Backtesting(default_conf) backtesting = Backtesting(default_conf)
dict_of_tickerrows = load_data_test('raise') dict_of_tickerrows = load_data_test('raise')
dataframes = backtesting.tickerdata_to_dataframe(dict_of_tickerrows) dataframes = backtesting.strategy.tickerdata_to_dataframe(dict_of_tickerrows)
dataframe = dataframes['UNITTEST/BTC'] dataframe = dataframes['UNITTEST/BTC']
cols = dataframe.columns cols = dataframe.columns
# assert the dataframe got some of the indicator columns # assert the dataframe got some of the indicator columns

View File

@ -194,7 +194,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog) -> None:
default_conf.update({'spaces': 'all'}) default_conf.update({'spaces': 'all'})
hyperopt = Hyperopt(default_conf) hyperopt = Hyperopt(default_conf)
hyperopt.tickerdata_to_dataframe = MagicMock() hyperopt.strategy.tickerdata_to_dataframe = MagicMock()
hyperopt.start() hyperopt.start()
parallel.assert_called_once() parallel.assert_called_once()
@ -242,7 +242,7 @@ def test_has_space(hyperopt):
def test_populate_indicators(hyperopt) -> None: def test_populate_indicators(hyperopt) -> None:
tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m')
tickerlist = {'UNITTEST/BTC': tick} tickerlist = {'UNITTEST/BTC': tick}
dataframes = hyperopt.tickerdata_to_dataframe(tickerlist) dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist)
dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'})
# Check if some indicators are generated. We will not test all of them # Check if some indicators are generated. We will not test all of them
@ -254,7 +254,7 @@ def test_populate_indicators(hyperopt) -> None:
def test_buy_strategy_generator(hyperopt) -> None: def test_buy_strategy_generator(hyperopt) -> None:
tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m')
tickerlist = {'UNITTEST/BTC': tick} tickerlist = {'UNITTEST/BTC': tick}
dataframes = hyperopt.tickerdata_to_dataframe(tickerlist) dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist)
dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'})
populate_buy_trend = hyperopt.buy_strategy_generator( populate_buy_trend = hyperopt.buy_strategy_generator(

View File

@ -7,7 +7,7 @@ from shutil import copyfile
import arrow import arrow
from freqtrade import optimize from freqtrade import optimize, constants
from freqtrade.arguments import TimeRange from freqtrade.arguments import TimeRange
from freqtrade.misc import file_dump_json from freqtrade.misc import file_dump_json
from freqtrade.optimize.__init__ import (download_backtesting_testdata, from freqtrade.optimize.__init__ import (download_backtesting_testdata,
@ -15,7 +15,8 @@ from freqtrade.optimize.__init__ import (download_backtesting_testdata,
load_cached_data_for_updating, load_cached_data_for_updating,
load_tickerdata_file, load_tickerdata_file,
make_testdata_path, trim_tickerlist) make_testdata_path, trim_tickerlist)
from freqtrade.tests.conftest import get_patched_exchange, log_has from freqtrade.strategy.default_strategy import DefaultStrategy
from freqtrade.tests.conftest import get_patched_exchange, log_has, patch_exchange
# Change this if modifying UNITTEST/BTC testdatafile # Change this if modifying UNITTEST/BTC testdatafile
_BTC_UNITTEST_LENGTH = 13681 _BTC_UNITTEST_LENGTH = 13681
@ -433,3 +434,61 @@ def test_file_dump_json() -> None:
# Remove the file # Remove the file
_clean_test_file(file) _clean_test_file(file)
def test_get_timeframe(default_conf, mocker) -> None:
patch_exchange(mocker)
strategy = DefaultStrategy(default_conf)
data = strategy.tickerdata_to_dataframe(
optimize.load_data(
None,
ticker_interval='1m',
pairs=['UNITTEST/BTC']
)
)
min_date, max_date = optimize.get_timeframe(data)
assert min_date.isoformat() == '2017-11-04T23:02:00+00:00'
assert max_date.isoformat() == '2017-11-14T22:58:00+00:00'
def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None:
patch_exchange(mocker)
strategy = DefaultStrategy(default_conf)
data = strategy.tickerdata_to_dataframe(
optimize.load_data(
None,
ticker_interval='1m',
pairs=['UNITTEST/BTC']
)
)
min_date, max_date = optimize.get_timeframe(data)
caplog.clear()
assert optimize.validate_backtest_data(data, min_date, max_date,
constants.TICKER_INTERVAL_MINUTES["1m"])
assert len(caplog.record_tuples) == 1
assert log_has(
"UNITTEST/BTC has missing frames: expected 14396, got 13680, that's 716 missing values",
caplog.record_tuples)
def test_validate_backtest_data(default_conf, mocker, caplog) -> None:
patch_exchange(mocker)
strategy = DefaultStrategy(default_conf)
timerange = TimeRange('index', 'index', 200, 250)
data = strategy.tickerdata_to_dataframe(
optimize.load_data(
None,
ticker_interval='5m',
pairs=['UNITTEST/BTC'],
timerange=timerange
)
)
min_date, max_date = optimize.get_timeframe(data)
caplog.clear()
assert not optimize.validate_backtest_data(data, min_date, max_date,
constants.TICKER_INTERVAL_MINUTES["5m"])
assert len(caplog.record_tuples) == 0

View File

@ -1,381 +0,0 @@
#!/usr/bin/env python3
"""
Script to display when the bot will buy a specific pair
Mandatory Cli parameters:
-p / --pair: pair to examine
Option but recommended
-s / --strategy: strategy to use
Optional Cli parameters
-d / --datadir: path to pair backtest data
--timerange: specify what timerange of data to use.
-l / --live: Live, to download the latest ticker for the pair
-db / --db-url: Show trades stored in database
Indicators recommended
Row 1: sma, ema3, ema5, ema10, ema50
Row 3: macd, rsi, fisher_rsi, mfi, slowd, slowk, fastd, fastk
Example of usage:
> python3 scripts/plot_dataframe.py --pair BTC/EUR -d user_data/data/ --indicators1 sma,ema3
--indicators2 fastk,fastd
"""
import json
import logging
import sys
from argparse import Namespace
from pathlib import Path
from typing import Dict, List, Any
import pandas as pd
import plotly.graph_objs as go
import pytz
from plotly import tools
from plotly.offline import plot
import freqtrade.optimize as optimize
from freqtrade import persistence
from freqtrade.arguments import Arguments, TimeRange
from freqtrade.exchange import Exchange
from freqtrade.optimize.backtesting import setup_configuration
from freqtrade.persistence import Trade
from freqtrade.strategy.resolver import StrategyResolver
logger = logging.getLogger(__name__)
_CONF: Dict[str, Any] = {}
timeZone = pytz.UTC
def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFrame:
trades: pd.DataFrame = pd.DataFrame()
if args.db_url:
persistence.init(_CONF)
columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"]
for x in Trade.query.all():
print("date: {}".format(x.open_date))
trades = pd.DataFrame([(t.pair, t.calc_profit(),
t.open_date.replace(tzinfo=timeZone),
t.close_date.replace(tzinfo=timeZone) if t.close_date else None,
t.open_rate, t.close_rate,
t.close_date.timestamp() - t.open_date.timestamp() if t.close_date else None)
for t in Trade.query.filter(Trade.pair.is_(pair)).all()],
columns=columns)
elif args.exportfilename:
file = Path(args.exportfilename)
# must align with columns in backtest.py
columns = ["pair", "profit", "opents", "closets", "index", "duration",
"open_rate", "close_rate", "open_at_end", "sell_reason"]
with file.open() as f:
data = json.load(f)
trades = pd.DataFrame(data, columns=columns)
trades = trades.loc[trades["pair"] == pair]
if timerange:
if timerange.starttype == 'date':
trades = trades.loc[trades["opents"] >= timerange.startts]
if timerange.stoptype == 'date':
trades = trades.loc[trades["opents"] <= timerange.stopts]
trades['opents'] = pd.to_datetime(trades['opents'],
unit='s',
utc=True,
infer_datetime_format=True)
trades['closets'] = pd.to_datetime(trades['closets'],
unit='s',
utc=True,
infer_datetime_format=True)
return trades
def plot_analyzed_dataframe(args: Namespace) -> None:
"""
Calls analyze() and plots the returned dataframe
:return: None
"""
global _CONF
# Load the configuration
_CONF.update(setup_configuration(args))
print(_CONF)
# Set the pair to audit
pair = args.pair
if pair is None:
logger.critical('Parameter --pair mandatory;. E.g --pair ETH/BTC')
exit()
if '/' not in pair:
logger.critical('--pair format must be XXX/YYY')
exit()
# Set timerange to use
timerange = Arguments.parse_timerange(args.timerange)
# Load the strategy
try:
strategy = StrategyResolver(_CONF).strategy
exchange = Exchange(_CONF)
except AttributeError:
logger.critical(
'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"',
args.strategy
)
exit()
# Set the ticker to use
tick_interval = strategy.ticker_interval
# Load pair tickers
tickers = {}
if args.live:
logger.info('Downloading pair.')
exchange.refresh_tickers([pair], tick_interval)
tickers[pair] = exchange.klines[pair]
else:
tickers = optimize.load_data(
datadir=_CONF.get("datadir"),
pairs=[pair],
ticker_interval=tick_interval,
refresh_pairs=_CONF.get('refresh_pairs', False),
timerange=timerange,
exchange=Exchange(_CONF)
)
# No ticker found, or impossible to download
if tickers == {}:
exit()
# Get trades already made from the DB
trades = load_trades(args, pair, timerange)
dataframes = strategy.tickerdata_to_dataframe(tickers)
dataframe = dataframes[pair]
dataframe = strategy.advise_buy(dataframe, {'pair': pair})
dataframe = strategy.advise_sell(dataframe, {'pair': pair})
if len(dataframe.index) > args.plot_limit:
logger.warning('Ticker contained more than %s candles as defined '
'with --plot-limit, clipping.', args.plot_limit)
dataframe = dataframe.tail(args.plot_limit)
trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']]
fig = generate_graph(
pair=pair,
trades=trades,
data=dataframe,
args=args
)
plot(fig, filename=str(Path('user_data').joinpath('freqtrade-plot.html')))
def generate_graph(pair, trades: pd.DataFrame, data: pd.DataFrame, args) -> tools.make_subplots:
"""
Generate the graph from the data generated by Backtesting or from DB
:param pair: Pair to Display on the graph
:param trades: All trades created
:param data: Dataframe
:param args: sys.argv that contrains the two params indicators1, and indicators2
:return: None
"""
# Define the graph
fig = tools.make_subplots(
rows=3,
cols=1,
shared_xaxes=True,
row_width=[1, 1, 4],
vertical_spacing=0.0001,
)
fig['layout'].update(title=pair)
fig['layout']['yaxis1'].update(title='Price')
fig['layout']['yaxis2'].update(title='Volume')
fig['layout']['yaxis3'].update(title='Other')
# Common information
candles = go.Candlestick(
x=data.date,
open=data.open,
high=data.high,
low=data.low,
close=data.close,
name='Price'
)
df_buy = data[data['buy'] == 1]
buys = go.Scattergl(
x=df_buy.date,
y=df_buy.close,
mode='markers',
name='buy',
marker=dict(
symbol='triangle-up-dot',
size=9,
line=dict(width=1),
color='green',
)
)
df_sell = data[data['sell'] == 1]
sells = go.Scattergl(
x=df_sell.date,
y=df_sell.close,
mode='markers',
name='sell',
marker=dict(
symbol='triangle-down-dot',
size=9,
line=dict(width=1),
color='red',
)
)
trade_buys = go.Scattergl(
x=trades["opents"],
y=trades["open_rate"],
mode='markers',
name='trade_buy',
marker=dict(
symbol='square-open',
size=11,
line=dict(width=2),
color='green'
)
)
trade_sells = go.Scattergl(
x=trades["closets"],
y=trades["close_rate"],
mode='markers',
name='trade_sell',
marker=dict(
symbol='square-open',
size=11,
line=dict(width=2),
color='red'
)
)
# Row 1
fig.append_trace(candles, 1, 1)
if 'bb_lowerband' in data and 'bb_upperband' in data:
bb_lower = go.Scatter(
x=data.date,
y=data.bb_lowerband,
name='BB lower',
line={'color': 'rgba(255,255,255,0)'},
)
bb_upper = go.Scatter(
x=data.date,
y=data.bb_upperband,
name='BB upper',
fill="tonexty",
fillcolor="rgba(0,176,246,0.2)",
line={'color': 'rgba(255,255,255,0)'},
)
fig.append_trace(bb_lower, 1, 1)
fig.append_trace(bb_upper, 1, 1)
fig = generate_row(fig=fig, row=1, raw_indicators=args.indicators1, data=data)
fig.append_trace(buys, 1, 1)
fig.append_trace(sells, 1, 1)
fig.append_trace(trade_buys, 1, 1)
fig.append_trace(trade_sells, 1, 1)
# Row 2
volume = go.Bar(
x=data['date'],
y=data['volume'],
name='Volume'
)
fig.append_trace(volume, 2, 1)
# Row 3
fig = generate_row(fig=fig, row=3, raw_indicators=args.indicators2, data=data)
return fig
def generate_row(fig, row, raw_indicators, data) -> tools.make_subplots:
"""
Generator all the indicator selected by the user for a specific row
"""
for indicator in raw_indicators.split(','):
if indicator in data:
scattergl = go.Scattergl(
x=data['date'],
y=data[indicator],
name=indicator
)
fig.append_trace(scattergl, row, 1)
else:
logger.info(
'Indicator "%s" ignored. Reason: This indicator is not found '
'in your strategy.',
indicator
)
return fig
def plot_parse_args(args: List[str]) -> Namespace:
"""
Parse args passed to the script
:param args: Cli arguments
:return: args: Array with all arguments
"""
arguments = Arguments(args, 'Graph dataframe')
arguments.scripts_options()
arguments.parser.add_argument(
'--indicators1',
help='Set indicators from your strategy you want in the first row of the graph. Separate '
'them with a coma. E.g: ema3,ema5 (default: %(default)s)',
type=str,
default='sma,ema3,ema5',
dest='indicators1',
)
arguments.parser.add_argument(
'--indicators2',
help='Set indicators from your strategy you want in the third row of the graph. Separate '
'them with a coma. E.g: fastd,fastk (default: %(default)s)',
type=str,
default='macd',
dest='indicators2',
)
arguments.parser.add_argument(
'--plot-limit',
help='Specify tick limit for plotting - too high values cause huge files - '
'Default: %(default)s',
dest='plot_limit',
default=750,
type=int,
)
arguments.common_args_parser()
arguments.optimizer_shared_options(arguments.parser)
arguments.backtesting_options(arguments.parser)
return arguments.parse_args()
def main(sysargv: List[str]) -> None:
"""
This function will initiate the bot and start the trading loop.
:return: None
"""
logger.info('Starting Plot Dataframe')
plot_analyzed_dataframe(
plot_parse_args(sysargv)
)
if __name__ == '__main__':
main(sys.argv[1:])