diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index f020252f8..cf2c52ed6 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -359,7 +359,7 @@ ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + "refresh_pairs", "live"]) ARGS_PLOT_PROFIT = (ARGS_COMMON + ARGS_STRATEGY + - ["pairs", "timerange", "export", "exportfilename"]) + ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source"]) class TimeRange(NamedTuple): diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 5a0dee042..dcd544d00 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -3,6 +3,7 @@ Helpers when analyzing backtest data """ import logging from pathlib import Path +from typing import Dict import numpy as np import pandas as pd @@ -101,6 +102,18 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: return trades +def load_trades(config) -> pd.DataFrame: + """ + Based on configuration option "trade_source": + * loads data from DB (using `db_url`) + * loads data from backtestfile (using `exportfilename`) + """ + if config["trade_source"] == "DB": + return load_trades_from_db(config["db_url"]) + elif config["trade_source"] == "file": + return load_backtest_data(Path(config["exportfilename"])) + + def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame: """ Compare trades and backtested pair DataFrames to get trades performed on backtested period @@ -109,3 +122,34 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> p trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) & (trades['close_time'] <= dataframe.iloc[-1]['date'])] return trades + + +def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame], column: str = "close"): + """ + Combine multiple dataframes "column" + :param tickers: Dict of Dataframes, dict key should be pair. + :param column: Column in the original dataframes to use + :return: DataFrame with the column renamed to the dict key, and a column + named mean, containing the mean of all pairs. + """ + df_comb = pd.concat([tickers[pair].set_index('date').rename( + {column: pair}, axis=1)[pair] for pair in tickers], axis=1) + + df_comb['mean'] = df_comb.mean(axis=1) + + return df_comb + + +def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str) -> pd.DataFrame: + """ + Adds a column `col_name` with the cumulative profit for the given trades array. + :param df: DataFrame with date index + :param trades: DataFrame containing trades (requires columns close_time and profitperc) + :return: Returns df with one additional column, col_name, containing the cumulative profit. + """ + df[col_name] = trades.set_index('close_time')['profitperc'].cumsum() + # Set first value to 0 + df.loc[df.iloc[0].name, col_name] = 0 + # FFill to get continuous + df[col_name] = df[col_name].ffill() + return df diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 460e20e91..05946e008 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -5,10 +5,8 @@ import gzip import logging import re from datetime import datetime -from typing import Dict import numpy as np -from pandas import DataFrame import rapidjson @@ -41,24 +39,6 @@ def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray: return dates.dt.to_pydatetime() -def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray: - """ - Return dates from Dataframe - :param dfs: Dict with format pair: pair_data - :return: List of dates - """ - alldates = {} - for pair, pair_data in dfs.items(): - dates = datesarray_to_datetimearray(pair_data['date']) - for date in dates: - alldates[date] = 1 - lst = [] - for date, _ in alldates.items(): - lst.append(date) - arr = np.array(lst) - return np.sort(arr, axis=0) - - def file_dump_json(filename, data, is_zip=False) -> None: """ Dump JSON data into a file diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index c058f7fb2..ccb932698 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -1,8 +1,15 @@ import logging -from typing import List +from pathlib import Path +from typing import Dict, List, Optional import pandas as pd -from pathlib import Path + +from freqtrade.arguments import Arguments +from freqtrade.data import history +from freqtrade.data.btanalysis import (combine_tickers_with_mean, + create_cum_profit, load_trades) +from freqtrade.exchange import Exchange +from freqtrade.resolvers import ExchangeResolver, StrategyResolver logger = logging.getLogger(__name__) @@ -16,7 +23,46 @@ except ImportError: exit(1) -def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots: +def init_plotscript(config): + """ + Initialize objects needed for plotting + :return: Dict with tickers, trades, pairs and strategy + """ + exchange: Optional[Exchange] = None + + # Exchange is only needed when downloading data! + if config.get("live", False) or config.get("refresh_pairs", False): + exchange = ExchangeResolver(config.get('exchange', {}).get('name'), + config).exchange + + strategy = StrategyResolver(config).strategy + if "pairs" in config: + pairs = config["pairs"].split(',') + else: + pairs = config["exchange"]["pair_whitelist"] + + # Set timerange to use + timerange = Arguments.parse_timerange(config["timerange"]) + + tickers = history.load_data( + datadir=Path(str(config.get("datadir"))), + pairs=pairs, + ticker_interval=config['ticker_interval'], + refresh_pairs=config.get('refresh_pairs', False), + timerange=timerange, + exchange=exchange, + live=config.get("live", False), + ) + + trades = load_trades(config) + return {"tickers": tickers, + "trades": trades, + "pairs": pairs, + "strategy": strategy, + } + + +def add_indicators(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots: """ Generator all the indicator selected by the user for a specific row :param fig: Plot figure to append to @@ -44,9 +90,29 @@ def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.m return fig -def plot_trades(fig, trades: pd.DataFrame): +def add_profit(fig, row, data: pd.DataFrame, column: str, name: str) -> tools.make_subplots: """ - Plot trades to "fig" + Add profit-plot + :param fig: Plot figure to append to + :param row: row number for this plot + :param data: candlestick DataFrame + :param column: Column to use for plot + :param name: Name to use + :return: fig with added profit plot + """ + profit = go.Scattergl( + x=data.index, + y=data[column], + name=name, + ) + fig.append_trace(profit, row, 1) + + return fig + + +def plot_trades(fig, trades: pd.DataFrame) -> tools.make_subplots: + """ + Add trades to "fig" """ # Trades can be empty if trades is not None and len(trades) > 0: @@ -86,13 +152,9 @@ def plot_trades(fig, trades: pd.DataFrame): return fig -def generate_graph( - pair: str, - data: pd.DataFrame, - trades: pd.DataFrame = None, - indicators1: List[str] = [], - indicators2: List[str] = [], -) -> go.Figure: +def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFrame = None, + indicators1: List[str] = [], + indicators2: List[str] = [],) -> go.Figure: """ Generate the graph from the data generated by Backtesting or from DB Volume will always be ploted in row2, so Row 1 and 3 are to our disposal for custom indicators @@ -186,7 +248,7 @@ def generate_graph( fig.append_trace(bb_upper, 1, 1) # Add indicators to main plot - fig = generate_row(fig=fig, row=1, indicators=indicators1, data=data) + fig = add_indicators(fig=fig, row=1, indicators=indicators1, data=data) fig = plot_trades(fig, trades) @@ -199,12 +261,54 @@ def generate_graph( fig.append_trace(volume, 2, 1) # Add indicators to seperate row - fig = generate_row(fig=fig, row=3, indicators=indicators2, data=data) + fig = add_indicators(fig=fig, row=3, indicators=indicators2, data=data) return fig -def generate_plot_file(fig, pair, ticker_interval) -> None: +def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], + trades: pd.DataFrame) -> go.Figure: + # Combine close-values for all pairs, rename columns to "pair" + df_comb = combine_tickers_with_mean(tickers, "close") + + # Add combined cumulative profit + df_comb = create_cum_profit(df_comb, trades, 'cum_profit') + + # Plot the pairs average close prices, and total profit growth + avgclose = go.Scattergl( + x=df_comb.index, + y=df_comb['mean'], + name='Avg close price', + ) + + fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1]) + fig['layout'].update(title="Profit plot") + + fig.append_trace(avgclose, 1, 1) + fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit') + + for pair in pairs: + profit_col = f'cum_profit_{pair}' + df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col) + + fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}") + + return fig + + +def generate_plot_filename(pair, ticker_interval) -> str: + """ + Generate filenames per pair/ticker_interval to be used for storing plots + """ + pair_name = pair.replace("/", "_") + file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html' + + logger.info('Generate plot file for %s', pair) + + return file_name + + +def store_plot_file(fig, filename: str, auto_open: bool = False) -> None: """ Generate a plot html file from pre populated fig plotly object :param fig: Plotly Figure to plot @@ -212,12 +316,8 @@ def generate_plot_file(fig, pair, ticker_interval) -> None: :param ticker_interval: Used as part of the filename :return: None """ - logger.info('Generate plot file for %s', pair) - - pair_name = pair.replace("/", "_") - file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html' Path("user_data/plots").mkdir(parents=True, exist_ok=True) - plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), - auto_open=False) + plot(fig, filename=str(Path('user_data/plots').joinpath(filename)), + auto_open=auto_open) diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index 1cb48393d..e8872f9a4 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -1,14 +1,18 @@ from unittest.mock import MagicMock -from arrow import Arrow import pytest +from arrow import Arrow from pandas import DataFrame, to_datetime -from freqtrade.arguments import TimeRange +from freqtrade.arguments import Arguments, TimeRange from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, + combine_tickers_with_mean, + create_cum_profit, extract_trades_of_period, - load_backtest_data, load_trades_from_db) -from freqtrade.data.history import load_pair_history, make_testdata_path + load_backtest_data, load_trades, + load_trades_from_db) +from freqtrade.data.history import (load_data, load_pair_history, + make_testdata_path) from freqtrade.tests.test_persistence import create_mock_trades @@ -74,3 +78,52 @@ def test_extract_trades_of_period(): assert trades1.iloc[0].close_time == Arrow(2017, 11, 14, 10, 41, 0).datetime assert trades1.iloc[-1].open_time == Arrow(2017, 11, 14, 14, 20, 0).datetime assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime + + +def test_load_trades(default_conf, mocker): + db_mock = mocker.patch("freqtrade.data.btanalysis.load_trades_from_db", MagicMock()) + bt_mock = mocker.patch("freqtrade.data.btanalysis.load_backtest_data", MagicMock()) + + default_conf['trade_source'] = "DB" + load_trades(default_conf) + + assert db_mock.call_count == 1 + assert bt_mock.call_count == 0 + + db_mock.reset_mock() + bt_mock.reset_mock() + default_conf['trade_source'] = "file" + default_conf['exportfilename'] = "testfile.json" + load_trades(default_conf) + + assert db_mock.call_count == 0 + assert bt_mock.call_count == 1 + + +def test_combine_tickers_with_mean(): + pairs = ["ETH/BTC", "XLM/BTC"] + tickers = load_data(datadir=None, + pairs=pairs, + ticker_interval='5m' + ) + df = combine_tickers_with_mean(tickers) + assert isinstance(df, DataFrame) + assert "ETH/BTC" in df.columns + assert "XLM/BTC" in df.columns + assert "mean" in df.columns + + +def test_create_cum_profit(): + filename = make_testdata_path(None) / "backtest-result_test.json" + bt_data = load_backtest_data(filename) + timerange = Arguments.parse_timerange("20180110-20180112") + + df = load_pair_history(pair="POWR/BTC", ticker_interval='5m', + datadir=None, timerange=timerange) + + cum_profits = create_cum_profit(df.set_index('date'), + bt_data[bt_data["pair"] == 'POWR/BTC'], + "cum_profits") + assert "cum_profits" in cum_profits.columns + assert cum_profits.iloc[0]['cum_profits'] == 0 + assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005 diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 7a7b15cf2..1a6b2a92d 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -4,10 +4,9 @@ import datetime from unittest.mock import MagicMock from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, - file_dump_json, file_load_json, format_ms_time, shorten_date) -from freqtrade.data.history import load_tickerdata_file, pair_data_filename -from freqtrade.strategy.default_strategy import DefaultStrategy +from freqtrade.data.history import pair_data_filename +from freqtrade.misc import (datesarray_to_datetimearray, file_dump_json, + file_load_json, format_ms_time, shorten_date) def test_shorten_date() -> None: @@ -32,20 +31,6 @@ def test_datesarray_to_datetimearray(ticker_history_list): assert date_len == 2 -def test_common_datearray(default_conf) -> None: - strategy = DefaultStrategy(default_conf) - tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, "1m", pair="UNITTEST/BTC", - fill_missing=True)} - dataframes = strategy.tickerdata_to_dataframe(tickerlist) - - dates = common_datearray(dataframes) - - assert dates.size == dataframes['UNITTEST/BTC']['date'].size - assert dates[0] == dataframes['UNITTEST/BTC']['date'][0] - assert dates[-1] == dataframes['UNITTEST/BTC']['date'].iloc[-1] - - def test_file_dump_json(mocker) -> None: file_open = mocker.patch('freqtrade.misc.open', MagicMock()) json_dump = mocker.patch('rapidjson.dump', MagicMock()) diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index ec81b93b8..cef229d19 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -1,21 +1,24 @@ +from copy import deepcopy from unittest.mock import MagicMock -from plotly import tools import plotly.graph_objs as go -from copy import deepcopy +from plotly import tools -from freqtrade.arguments import TimeRange +from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history -from freqtrade.data.btanalysis import load_backtest_data -from freqtrade.plot.plotting import (generate_graph, generate_plot_file, - generate_row, plot_trades) +from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data +from freqtrade.plot.plotting import (add_indicators, add_profit, + generate_candlestick_graph, + generate_plot_filename, + generate_profit_graph, init_plotscript, + plot_trades, store_plot_file) from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.tests.conftest import log_has, log_has_re def fig_generating_mock(fig, *args, **kwargs): - """ Return Fig - used to mock generate_row and plot_trades""" + """ Return Fig - used to mock add_indicators and plot_trades""" return fig @@ -34,7 +37,27 @@ def generage_empty_figure(): ) -def test_generate_row(default_conf, caplog): +def test_init_plotscript(default_conf, mocker): + default_conf['timerange'] = "20180110-20180112" + default_conf['trade_source'] = "file" + default_conf['ticker_interval'] = "5m" + default_conf["datadir"] = history.make_testdata_path(None) + default_conf['exportfilename'] = str( + history.make_testdata_path(None) / "backtest-result_test.json") + ret = init_plotscript(default_conf) + assert "tickers" in ret + assert "trades" in ret + assert "pairs" in ret + assert "strategy" in ret + + default_conf['pairs'] = "POWR/BTC,XLM/BTC" + ret = init_plotscript(default_conf) + assert "tickers" in ret + assert "POWR/BTC" in ret["tickers"] + assert "XLM/BTC" in ret["tickers"] + + +def test_add_indicators(default_conf, caplog): pair = "UNITTEST/BTC" timerange = TimeRange(None, 'line', 0, -1000) @@ -49,20 +72,20 @@ def test_generate_row(default_conf, caplog): fig = generage_empty_figure() # Row 1 - fig1 = generate_row(fig=deepcopy(fig), row=1, indicators=indicators1, data=data) + fig1 = add_indicators(fig=deepcopy(fig), row=1, indicators=indicators1, data=data) figure = fig1.layout.figure ema10 = find_trace_in_fig_data(figure.data, "ema10") assert isinstance(ema10, go.Scatter) assert ema10.yaxis == "y" - fig2 = generate_row(fig=deepcopy(fig), row=3, indicators=indicators2, data=data) + fig2 = add_indicators(fig=deepcopy(fig), row=3, indicators=indicators2, data=data) figure = fig2.layout.figure macd = find_trace_in_fig_data(figure.data, "macd") assert isinstance(macd, go.Scatter) assert macd.yaxis == "y3" # No indicator found - fig3 = generate_row(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data) + fig3 = add_indicators(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data) assert fig == fig3 assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog.record_tuples) @@ -95,8 +118,8 @@ def test_plot_trades(caplog): assert trade_sell.marker.color == 'red' -def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog): - row_mock = mocker.patch('freqtrade.plot.plotting.generate_row', +def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, caplog): + row_mock = mocker.patch('freqtrade.plot.plotting.add_indicators', MagicMock(side_effect=fig_generating_mock)) trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades', MagicMock(side_effect=fig_generating_mock)) @@ -110,8 +133,8 @@ def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog): indicators1 = [] indicators2 = [] - fig = generate_graph(pair=pair, data=data, trades=None, - indicators1=indicators1, indicators2=indicators2) + fig = generate_candlestick_graph(pair=pair, data=data, trades=None, + indicators1=indicators1, indicators2=indicators2) assert isinstance(fig, go.Figure) assert fig.layout.title.text == pair figure = fig.layout.figure @@ -131,8 +154,8 @@ def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog): assert log_has("No sell-signals found.", caplog.record_tuples) -def test_generate_graph_no_trades(default_conf, mocker): - row_mock = mocker.patch('freqtrade.plot.plotting.generate_row', +def test_generate_candlestick_graph_no_trades(default_conf, mocker): + row_mock = mocker.patch('freqtrade.plot.plotting.add_indicators', MagicMock(side_effect=fig_generating_mock)) trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades', MagicMock(side_effect=fig_generating_mock)) @@ -147,8 +170,8 @@ def test_generate_graph_no_trades(default_conf, mocker): indicators1 = [] indicators2 = [] - fig = generate_graph(pair=pair, data=data, trades=None, - indicators1=indicators1, indicators2=indicators2) + fig = generate_candlestick_graph(pair=pair, data=data, trades=None, + indicators1=indicators1, indicators2=indicators2) assert isinstance(fig, go.Figure) assert fig.layout.title.text == pair figure = fig.layout.figure @@ -178,12 +201,68 @@ def test_generate_graph_no_trades(default_conf, mocker): assert trades_mock.call_count == 1 +def test_generate_Plot_filename(): + fn = generate_plot_filename("UNITTEST/BTC", "5m") + assert fn == "freqtrade-plot-UNITTEST_BTC-5m.html" + + def test_generate_plot_file(mocker, caplog): fig = generage_empty_figure() plot_mock = mocker.patch("freqtrade.plot.plotting.plot", MagicMock()) - generate_plot_file(fig, "UNITTEST/BTC", "5m") + store_plot_file(fig, filename="freqtrade-plot-UNITTEST_BTC-5m.html") assert plot_mock.call_count == 1 assert plot_mock.call_args[0][0] == fig assert (plot_mock.call_args_list[0][1]['filename'] == "user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html") + + +def test_add_profit(): + filename = history.make_testdata_path(None) / "backtest-result_test.json" + bt_data = load_backtest_data(filename) + timerange = Arguments.parse_timerange("20180110-20180112") + + df = history.load_pair_history(pair="POWR/BTC", ticker_interval='5m', + datadir=None, timerange=timerange) + fig = generage_empty_figure() + + cum_profits = create_cum_profit(df.set_index('date'), + bt_data[bt_data["pair"] == 'POWR/BTC'], + "cum_profits") + + fig1 = add_profit(fig, row=2, data=cum_profits, column='cum_profits', name='Profits') + figure = fig1.layout.figure + profits = find_trace_in_fig_data(figure.data, "Profits") + assert isinstance(profits, go.Scattergl) + assert profits.yaxis == "y2" + + +def test_generate_profit_graph(): + filename = history.make_testdata_path(None) / "backtest-result_test.json" + trades = load_backtest_data(filename) + timerange = Arguments.parse_timerange("20180110-20180112") + pairs = ["POWR/BTC", "XLM/BTC"] + + tickers = history.load_data(datadir=None, + pairs=pairs, + ticker_interval='5m', + timerange=timerange + ) + trades = trades[trades['pair'].isin(pairs)] + + fig = generate_profit_graph(pairs, tickers, trades) + assert isinstance(fig, go.Figure) + + assert fig.layout.title.text == "Profit plot" + figure = fig.layout.figure + assert len(figure.data) == 4 + + avgclose = find_trace_in_fig_data(figure.data, "Avg close price") + assert isinstance(avgclose, go.Scattergl) + + profit = find_trace_in_fig_data(figure.data, "Profit") + assert isinstance(profit, go.Scattergl) + + for pair in pairs: + profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}") + assert isinstance(profit_pair, go.Scattergl) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 7e81af925..1e2d9f248 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -2,19 +2,7 @@ """ Script to display when the bot will buy on specific pair(s) -Mandatory Cli parameters: --p / --pairs: pair(s) to examine - -Option but recommended --s / --strategy: strategy to use - - -Optional Cli parameters --d / --datadir: path to pair(s) backtest data ---timerange: specify what timerange of data to use. --l / --live: Live, to download the latest ticker for the pair(s) --db / --db-url: Show trades stored in database - +Use `python plot_dataframe.py --help` to display the command line arguments Indicators recommended Row 1: sma, ema3, ema5, ema10, ema50 @@ -26,18 +14,16 @@ Example of usage: """ import logging import sys -from pathlib import Path from typing import Any, Dict, List import pandas as pd from freqtrade.arguments import ARGS_PLOT_DATAFRAME, Arguments -from freqtrade.data import history -from freqtrade.data.btanalysis import (extract_trades_of_period, - load_backtest_data, load_trades_from_db) +from freqtrade.data.btanalysis import extract_trades_of_period from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import generate_graph, generate_plot_file -from freqtrade.resolvers import ExchangeResolver, StrategyResolver +from freqtrade.plot.plotting import (init_plotscript, generate_candlestick_graph, + store_plot_file, + generate_plot_filename) from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -68,52 +54,29 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): -Generate plot files :return: None """ - exchange = ExchangeResolver(config.get('exchange', {}).get('name'), config).exchange - - strategy = StrategyResolver(config).strategy - if "pairs" in config: - pairs = config["pairs"].split(',') - else: - pairs = config["exchange"]["pair_whitelist"] - - # Set timerange to use - timerange = Arguments.parse_timerange(config["timerange"]) - ticker_interval = strategy.ticker_interval - - tickers = history.load_data( - datadir=Path(str(config.get("datadir"))), - pairs=pairs, - ticker_interval=config['ticker_interval'], - refresh_pairs=config.get('refresh_pairs', False), - timerange=timerange, - exchange=exchange, - live=config.get("live", False), - ) + plot_elements = init_plotscript(config) + trades = plot_elements['trades'] pair_counter = 0 - for pair, data in tickers.items(): + for pair, data in plot_elements["tickers"].items(): pair_counter += 1 logger.info("analyse pair %s", pair) tickers = {} tickers[pair] = data - dataframe = generate_dataframe(strategy, tickers, pair) - if config["trade_source"] == "DB": - trades = load_trades_from_db(config["db_url"]) - elif config["trade_source"] == "file": - trades = load_backtest_data(Path(config["exportfilename"])) + dataframe = generate_dataframe(plot_elements["strategy"], tickers, pair) - trades = trades.loc[trades['pair'] == pair] - trades = extract_trades_of_period(dataframe, trades) + trades_pair = trades.loc[trades['pair'] == pair] + trades_pair = extract_trades_of_period(dataframe, trades_pair) - fig = generate_graph( + fig = generate_candlestick_graph( pair=pair, data=dataframe, - trades=trades, + trades=trades_pair, indicators1=config["indicators1"].split(","), indicators2=config["indicators2"].split(",") ) - generate_plot_file(fig, pair, ticker_interval) + store_plot_file(fig, generate_plot_filename(pair, config['ticker_interval'])) logger.info('End of ploting process %s plots generated', pair_counter) @@ -130,7 +93,7 @@ def plot_parse_args(args: List[str]) -> Dict[str, Any]: parsed_args = arguments.parse_args() # Load the configuration - config = setup_configuration(parsed_args, RunMode.BACKTEST) + config = setup_configuration(parsed_args, RunMode.OTHER) return config diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 32bfae9cc..7442ef155 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -2,204 +2,39 @@ """ Script to display profits -Mandatory Cli parameters: --p / --pair: pair to examine - -Optional Cli parameters --c / --config: specify configuration file --s / --strategy: strategy to use --d / --datadir: path to pair backtest data ---timerange: specify what timerange of data to use ---export-filename: Specify where the backtest export is located. +Use `python plot_profit.py --help` to display the command line arguments """ -import json import logging import sys -from argparse import Namespace -from pathlib import Path -from typing import List, Optional +from typing import Any, Dict, List -import numpy as np -import plotly.graph_objs as go -from plotly import tools -from plotly.offline import plot - -from freqtrade.arguments import Arguments, ARGS_PLOT_PROFIT -from freqtrade.configuration import Configuration -from freqtrade.data import history -from freqtrade.exchange import timeframe_to_seconds -from freqtrade.misc import common_datearray -from freqtrade.resolvers import StrategyResolver +from freqtrade.arguments import ARGS_PLOT_PROFIT, Arguments +from freqtrade.optimize import setup_configuration +from freqtrade.plot.plotting import init_plotscript, generate_profit_graph, store_plot_file from freqtrade.state import RunMode - logger = logging.getLogger(__name__) -# data:: [ pair, profit-%, enter, exit, time, duration] -# data:: ["ETH/BTC", 0.0023975, "1515598200", "1515602100", "2018-01-10 07:30:00+00:00", 65] -def make_profit_array(data: List, px: int, min_date: int, - interval: str, - filter_pairs: Optional[List] = None) -> np.ndarray: - pg = np.zeros(px) - filter_pairs = filter_pairs or [] - # Go through the trades - # and make an total profit - # array - for trade in data: - pair = trade[0] - if filter_pairs and pair not in filter_pairs: - continue - profit = trade[1] - trade_sell_time = int(trade[3]) - - ix = define_index(min_date, trade_sell_time, interval) - if ix < px: - logger.debug('[%s]: Add profit %s on %s', pair, profit, trade[4]) - pg[ix] += profit - - # rewrite the pg array to go from - # total profits at each timeframe - # to accumulated profits - pa = 0 - for x in range(0, len(pg)): - p = pg[x] # Get current total percent - pa += p # Add to the accumulated percent - pg[x] = pa # write back to save memory - - return pg - - -def plot_profit(args: Namespace) -> None: +def plot_profit(config: Dict[str, Any]) -> None: """ Plots the total profit for all pairs. Note, the profit calculation isn't realistic. But should be somewhat proportional, and therefor useful in helping out to find a good algorithm. """ + plot_elements = init_plotscript(config) + trades = plot_elements['trades'] + # Filter trades to relevant pairs + trades = trades[trades['pair'].isin(plot_elements["pairs"])] - # We need to use the same pairs, same ticker_interval - # and same timeperiod as used in backtesting - # to match the tickerdata against the profits-results - timerange = Arguments.parse_timerange(args.timerange) - - config = Configuration(args, RunMode.OTHER).get_config() - - # Init strategy - try: - strategy = StrategyResolver({'strategy': config.get('strategy')}).strategy - - except AttributeError: - logger.critical( - 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', - config.get('strategy') - ) - exit(1) - - # Load the profits results - try: - filename = args.exportfilename - with open(filename) as file: - data = json.load(file) - except FileNotFoundError: - logger.critical( - 'File "backtest-result.json" not found. This script require backtesting ' - 'results to run.\nPlease run a backtesting with the parameter --export.') - exit(1) - - # Take pairs from the cli otherwise switch to the pair in the config file - if args.pairs: - filter_pairs = args.pairs - filter_pairs = filter_pairs.split(',') - else: - filter_pairs = config['exchange']['pair_whitelist'] - - ticker_interval = strategy.ticker_interval - pairs = config['exchange']['pair_whitelist'] - - if filter_pairs: - pairs = list(set(pairs) & set(filter_pairs)) - logger.info('Filter, keep pairs %s' % pairs) - - tickers = history.load_data( - datadir=Path(str(config.get('datadir'))), - pairs=pairs, - ticker_interval=ticker_interval, - refresh_pairs=False, - timerange=timerange - ) - dataframes = strategy.tickerdata_to_dataframe(tickers) - - # NOTE: the dataframes are of unequal length, - # 'dates' is an merged date array of them all. - - dates = common_datearray(dataframes) - min_date = int(min(dates).timestamp()) - max_date = int(max(dates).timestamp()) - num_iterations = define_index(min_date, max_date, ticker_interval) + 1 - - # Make an average close price of all the pairs that was involved. + # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend - # We are essentially saying: - # array <- sum dataframes[*]['close'] / num_items dataframes - # FIX: there should be some onliner numpy/panda for this - avgclose = np.zeros(num_iterations) - num = 0 - for pair, pair_data in dataframes.items(): - close = pair_data['close'] - maxprice = max(close) # Normalize price to [0,1] - logger.info('Pair %s has length %s' % (pair, len(close))) - for x in range(0, len(close)): - avgclose[x] += close[x] / maxprice - # avgclose += close - num += 1 - avgclose /= num - - # make an profits-growth array - pg = make_profit_array(data, num_iterations, min_date, ticker_interval, filter_pairs) - - # - # Plot the pairs average close prices, and total profit growth - # - - avgclose = go.Scattergl( - x=dates, - y=avgclose, - name='Avg close price', - ) - - profit = go.Scattergl( - x=dates, - y=pg, - name='Profit', - ) - - fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1]) - - fig.append_trace(avgclose, 1, 1) - fig.append_trace(profit, 2, 1) - - for pair in pairs: - pg = make_profit_array(data, num_iterations, min_date, ticker_interval, [pair]) - pair_profit = go.Scattergl( - x=dates, - y=pg, - name=pair, - ) - fig.append_trace(pair_profit, 3, 1) - - plot(fig, filename=str(Path('user_data').joinpath('freqtrade-profit-plot.html'))) + fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], trades) + store_plot_file(fig, filename='freqtrade-profit-plot.html', auto_open=True) -def define_index(min_date: int, max_date: int, ticker_interval: str) -> int: - """ - Return the index of a specific date - """ - interval_seconds = timeframe_to_seconds(ticker_interval) - return int((max_date - min_date) / interval_seconds) - - -def plot_parse_args(args: List[str]) -> Namespace: +def plot_parse_args(args: List[str]) -> Dict[str, Any]: """ Parse args passed to the script :param args: Cli arguments @@ -208,7 +43,11 @@ def plot_parse_args(args: List[str]) -> Namespace: arguments = Arguments(args, 'Graph profits') arguments.build_args(optionlist=ARGS_PLOT_PROFIT) - return arguments.parse_args() + parsed_args = arguments.parse_args() + + # Load the configuration + config = setup_configuration(parsed_args, RunMode.OTHER) + return config def main(sysargv: List[str]) -> None: