From 0436811cf06f44a77f4164d5daa2521a72a99f5c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 Jun 2019 06:47:40 +0200 Subject: [PATCH 01/25] Use mode OTHER, nto backtesting --- scripts/plot_dataframe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 7e81af925..232fb7aad 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -130,7 +130,7 @@ def plot_parse_args(args: List[str]) -> Dict[str, Any]: parsed_args = arguments.parse_args() # Load the configuration - config = setup_configuration(parsed_args, RunMode.BACKTEST) + config = setup_configuration(parsed_args, RunMode.OTHER) return config From 044be3b93e0c51c0826a0205219645ed2e4c32f8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Jun 2019 16:57:04 +0200 Subject: [PATCH 02/25] Add create_cum_profit column --- freqtrade/arguments.py | 2 +- freqtrade/data/btanalysis.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index f020252f8..cf2c52ed6 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -359,7 +359,7 @@ ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + "refresh_pairs", "live"]) ARGS_PLOT_PROFIT = (ARGS_COMMON + ARGS_STRATEGY + - ["pairs", "timerange", "export", "exportfilename"]) + ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source"]) class TimeRange(NamedTuple): diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 5a0dee042..30fd5bc93 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -109,3 +109,15 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> p trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) & (trades['close_time'] <= dataframe.iloc[-1]['date'])] return trades + + +def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str): + """ + Adds a column `col_name` with the cumulative profit for the given trades array. + """ + df[col_name] = trades.set_index('close_time')['profitperc'].cumsum() + # Set first value to 0 + df.loc[df.iloc[0].name, col_name] = 0 + # FFill to get continuous + df[col_name] = df[col_name].ffill() + return df From edd3fc88256d8322abda7284d35165d3079021db Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Jun 2019 17:19:42 +0200 Subject: [PATCH 03/25] Add test for create_cum_profit --- freqtrade/data/btanalysis.py | 3 +++ freqtrade/tests/data/test_btanalysis.py | 22 +++++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 30fd5bc93..dae891423 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -114,6 +114,9 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> p def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str): """ Adds a column `col_name` with the cumulative profit for the given trades array. + :param df: DataFrame with date index + :param trades: DataFrame containing trades (requires columns close_time and profitperc) + :return: Returns df with one additional column, col_name, containing the cumulative profit. """ df[col_name] = trades.set_index('close_time')['profitperc'].cumsum() # Set first value to 0 diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index 1cb48393d..4eca73934 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -1,11 +1,11 @@ from unittest.mock import MagicMock -from arrow import Arrow import pytest +from arrow import Arrow from pandas import DataFrame, to_datetime -from freqtrade.arguments import TimeRange -from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, +from freqtrade.arguments import TimeRange, Arguments +from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, create_cum_profit, extract_trades_of_period, load_backtest_data, load_trades_from_db) from freqtrade.data.history import load_pair_history, make_testdata_path @@ -74,3 +74,19 @@ def test_extract_trades_of_period(): assert trades1.iloc[0].close_time == Arrow(2017, 11, 14, 10, 41, 0).datetime assert trades1.iloc[-1].open_time == Arrow(2017, 11, 14, 14, 20, 0).datetime assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime + + +def test_create_cum_profit(): + filename = make_testdata_path(None) / "backtest-result_test.json" + bt_data = load_backtest_data(filename) + timerange = Arguments.parse_timerange("20180110-20180112") + + df = load_pair_history(pair="POWR/BTC", ticker_interval='5m', + datadir=None, timerange=timerange) + + cum_profits = create_cum_profit(df.set_index('date'), + bt_data[bt_data["pair"] == 'POWR/BTC'], + "cum_profits") + assert "cum_profits" in cum_profits.columns + assert cum_profits.iloc[0]['cum_profits'] == 0 + assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005 From 79b4e2dc8596bd0424e551e29b9f333f5d099439 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Jun 2019 17:23:03 +0200 Subject: [PATCH 04/25] Rename generate_graph to generate_candlestick_graph --- freqtrade/plot/plotting.py | 2 +- freqtrade/tests/test_plotting.py | 14 +++++++------- scripts/plot_dataframe.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index c058f7fb2..ae9889975 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -86,7 +86,7 @@ def plot_trades(fig, trades: pd.DataFrame): return fig -def generate_graph( +def generate_candlestick_graph( pair: str, data: pd.DataFrame, trades: pd.DataFrame = None, diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index ec81b93b8..46462bd76 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -8,7 +8,7 @@ from copy import deepcopy from freqtrade.arguments import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import load_backtest_data -from freqtrade.plot.plotting import (generate_graph, generate_plot_file, +from freqtrade.plot.plotting import (generate_candlestick_graph, generate_plot_file, generate_row, plot_trades) from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.tests.conftest import log_has, log_has_re @@ -95,7 +95,7 @@ def test_plot_trades(caplog): assert trade_sell.marker.color == 'red' -def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog): +def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, caplog): row_mock = mocker.patch('freqtrade.plot.plotting.generate_row', MagicMock(side_effect=fig_generating_mock)) trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades', @@ -110,8 +110,8 @@ def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog): indicators1 = [] indicators2 = [] - fig = generate_graph(pair=pair, data=data, trades=None, - indicators1=indicators1, indicators2=indicators2) + fig = generate_candlestick_graph(pair=pair, data=data, trades=None, + indicators1=indicators1, indicators2=indicators2) assert isinstance(fig, go.Figure) assert fig.layout.title.text == pair figure = fig.layout.figure @@ -131,7 +131,7 @@ def test_generate_graph_no_signals_no_trades(default_conf, mocker, caplog): assert log_has("No sell-signals found.", caplog.record_tuples) -def test_generate_graph_no_trades(default_conf, mocker): +def test_generate_candlestick_graph_no_trades(default_conf, mocker): row_mock = mocker.patch('freqtrade.plot.plotting.generate_row', MagicMock(side_effect=fig_generating_mock)) trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades', @@ -147,8 +147,8 @@ def test_generate_graph_no_trades(default_conf, mocker): indicators1 = [] indicators2 = [] - fig = generate_graph(pair=pair, data=data, trades=None, - indicators1=indicators1, indicators2=indicators2) + fig = generate_candlestick_graph(pair=pair, data=data, trades=None, + indicators1=indicators1, indicators2=indicators2) assert isinstance(fig, go.Figure) assert fig.layout.title.text == pair figure = fig.layout.figure diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 232fb7aad..80773b3b0 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -36,7 +36,7 @@ from freqtrade.data import history from freqtrade.data.btanalysis import (extract_trades_of_period, load_backtest_data, load_trades_from_db) from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import generate_graph, generate_plot_file +from freqtrade.plot.plotting import generate_candlestick_graph, generate_plot_file from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode @@ -105,7 +105,7 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): trades = trades.loc[trades['pair'] == pair] trades = extract_trades_of_period(dataframe, trades) - fig = generate_graph( + fig = generate_candlestick_graph( pair=pair, data=dataframe, trades=trades, From 4506832925717d59b2d307a296996768efa3da8c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Jun 2019 20:07:25 +0200 Subject: [PATCH 05/25] Update docstring --- scripts/plot_dataframe.py | 14 +------------- scripts/plot_profit.py | 10 +--------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 80773b3b0..701672f29 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -2,19 +2,7 @@ """ Script to display when the bot will buy on specific pair(s) -Mandatory Cli parameters: --p / --pairs: pair(s) to examine - -Option but recommended --s / --strategy: strategy to use - - -Optional Cli parameters --d / --datadir: path to pair(s) backtest data ---timerange: specify what timerange of data to use. --l / --live: Live, to download the latest ticker for the pair(s) --db / --db-url: Show trades stored in database - +Use `python plot_dataframe.py --help` to display the command line arguments Indicators recommended Row 1: sma, ema3, ema5, ema10, ema50 diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 32bfae9cc..01dc260d9 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -2,15 +2,7 @@ """ Script to display profits -Mandatory Cli parameters: --p / --pair: pair to examine - -Optional Cli parameters --c / --config: specify configuration file --s / --strategy: strategy to use --d / --datadir: path to pair backtest data ---timerange: specify what timerange of data to use ---export-filename: Specify where the backtest export is located. +Use `python plot_profit.py --help` to display the command line arguments """ import json import logging From e50eee59cf5c5450ff035eaabce0c025dd1580ca Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Jun 2019 20:30:31 +0200 Subject: [PATCH 06/25] Seperate plot-name generation and plotting --- freqtrade/plot/plotting.py | 19 ++++++++++++------- freqtrade/tests/test_plotting.py | 17 ++++++++++++----- scripts/plot_dataframe.py | 6 ++++-- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index ae9889975..f8a010068 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -204,7 +204,16 @@ def generate_candlestick_graph( return fig -def generate_plot_file(fig, pair, ticker_interval) -> None: +def generate_plot_filename(pair, ticker_interval) -> str: + pair_name = pair.replace("/", "_") + file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html' + + logger.info('Generate plot file for %s', pair) + + return file_name + + +def generate_plot_file(fig, filename: str, auto_open: bool = False) -> None: """ Generate a plot html file from pre populated fig plotly object :param fig: Plotly Figure to plot @@ -212,12 +221,8 @@ def generate_plot_file(fig, pair, ticker_interval) -> None: :param ticker_interval: Used as part of the filename :return: None """ - logger.info('Generate plot file for %s', pair) - - pair_name = pair.replace("/", "_") - file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html' Path("user_data/plots").mkdir(parents=True, exist_ok=True) - plot(fig, filename=str(Path('user_data/plots').joinpath(file_name)), - auto_open=False) + plot(fig, filename=str(Path('user_data/plots').joinpath(filename)), + auto_open=auto_open) diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index 46462bd76..527534522 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -1,15 +1,17 @@ +from copy import deepcopy from unittest.mock import MagicMock -from plotly import tools import plotly.graph_objs as go -from copy import deepcopy +from plotly import tools from freqtrade.arguments import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import load_backtest_data -from freqtrade.plot.plotting import (generate_candlestick_graph, generate_plot_file, - generate_row, plot_trades) +from freqtrade.plot.plotting import (generate_candlestick_graph, + generate_plot_file, + generate_plot_filename, generate_row, + plot_trades) from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.tests.conftest import log_has, log_has_re @@ -178,10 +180,15 @@ def test_generate_candlestick_graph_no_trades(default_conf, mocker): assert trades_mock.call_count == 1 +def test_generate_Plot_filename(): + fn = generate_plot_filename("UNITTEST/BTC", "5m") + assert fn == "freqtrade-plot-UNITTEST_BTC-5m.html" + + def test_generate_plot_file(mocker, caplog): fig = generage_empty_figure() plot_mock = mocker.patch("freqtrade.plot.plotting.plot", MagicMock()) - generate_plot_file(fig, "UNITTEST/BTC", "5m") + generate_plot_file(fig, filename="freqtrade-plot-UNITTEST_BTC-5m.html") assert plot_mock.call_count == 1 assert plot_mock.call_args[0][0] == fig diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 701672f29..d97c6f041 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -24,7 +24,9 @@ from freqtrade.data import history from freqtrade.data.btanalysis import (extract_trades_of_period, load_backtest_data, load_trades_from_db) from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import generate_candlestick_graph, generate_plot_file +from freqtrade.plot.plotting import (generate_candlestick_graph, + generate_plot_file, + generate_plot_filename) from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode @@ -101,7 +103,7 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): indicators2=config["indicators2"].split(",") ) - generate_plot_file(fig, pair, ticker_interval) + generate_plot_file(fig, generate_plot_filename(pair, ticker_interval)) logger.info('End of ploting process %s plots generated', pair_counter) From 4218d569de7dfa3e400d0f4f7c2dd4698c6961f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Jun 2019 20:41:22 +0200 Subject: [PATCH 07/25] Only read trades once --- scripts/plot_dataframe.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index d97c6f041..07079304e 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -80,6 +80,11 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): live=config.get("live", False), ) + if config["trade_source"] == "DB": + trades = load_trades_from_db(config["db_url"]) + elif config["trade_source"] == "file": + trades = load_backtest_data(Path(config["exportfilename"])) + pair_counter = 0 for pair, data in tickers.items(): pair_counter += 1 @@ -87,18 +92,14 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): tickers = {} tickers[pair] = data dataframe = generate_dataframe(strategy, tickers, pair) - if config["trade_source"] == "DB": - trades = load_trades_from_db(config["db_url"]) - elif config["trade_source"] == "file": - trades = load_backtest_data(Path(config["exportfilename"])) - trades = trades.loc[trades['pair'] == pair] - trades = extract_trades_of_period(dataframe, trades) + trades_pair = trades.loc[trades['pair'] == pair] + trades_pair = extract_trades_of_period(dataframe, trades_pair) fig = generate_candlestick_graph( pair=pair, data=dataframe, - trades=trades, + trades=trades_pair, indicators1=config["indicators1"].split(","), indicators2=config["indicators2"].split(",") ) From 8aa327cb8a331119a919d217e3ea29beadad08b0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Jun 2019 20:50:31 +0200 Subject: [PATCH 08/25] Add load_trades abstraction (to load trades from either DB or file) --- freqtrade/data/btanalysis.py | 14 ++++++++++++++ freqtrade/tests/data/test_btanalysis.py | 25 +++++++++++++++++++++++-- scripts/plot_dataframe.py | 8 ++------ 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index dae891423..f6f7e5541 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -101,6 +101,20 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: return trades +def load_trades(config) -> pd.DataFrame: + """ + Based on configuration option "trade_source": + * loads data from DB (using `db_url`) + * loads data from backtestfile (`using exportfilename`) + """ + if config["trade_source"] == "DB": + return load_trades_from_db(config["db_url"]) + elif config["trade_source"] == "file": + return load_backtest_data(Path(config["exportfilename"])) + else: + return None + + def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame: """ Compare trades and backtested pair DataFrames to get trades performed on backtested period diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index 4eca73934..01e5dc90d 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -4,10 +4,11 @@ import pytest from arrow import Arrow from pandas import DataFrame, to_datetime -from freqtrade.arguments import TimeRange, Arguments +from freqtrade.arguments import Arguments, TimeRange from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, create_cum_profit, extract_trades_of_period, - load_backtest_data, load_trades_from_db) + load_backtest_data, load_trades, + load_trades_from_db) from freqtrade.data.history import load_pair_history, make_testdata_path from freqtrade.tests.test_persistence import create_mock_trades @@ -76,6 +77,26 @@ def test_extract_trades_of_period(): assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime +def test_load_trades(default_conf, mocker): + db_mock = mocker.patch("freqtrade.data.btanalysis.load_trades_from_db", MagicMock()) + bt_mock = mocker.patch("freqtrade.data.btanalysis.load_backtest_data", MagicMock()) + + default_conf['trade_source'] = "DB" + load_trades(default_conf) + + assert db_mock.call_count == 1 + assert bt_mock.call_count == 0 + + db_mock.reset_mock() + bt_mock.reset_mock() + default_conf['trade_source'] = "file" + default_conf['exportfilename'] = "testfile.json" + load_trades(default_conf) + + assert db_mock.call_count == 0 + assert bt_mock.call_count == 1 + + def test_create_cum_profit(): filename = make_testdata_path(None) / "backtest-result_test.json" bt_data = load_backtest_data(filename) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 07079304e..dfd31e9ab 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -21,8 +21,7 @@ import pandas as pd from freqtrade.arguments import ARGS_PLOT_DATAFRAME, Arguments from freqtrade.data import history -from freqtrade.data.btanalysis import (extract_trades_of_period, - load_backtest_data, load_trades_from_db) +from freqtrade.data.btanalysis import extract_trades_of_period, load_trades from freqtrade.optimize import setup_configuration from freqtrade.plot.plotting import (generate_candlestick_graph, generate_plot_file, @@ -80,10 +79,7 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): live=config.get("live", False), ) - if config["trade_source"] == "DB": - trades = load_trades_from_db(config["db_url"]) - elif config["trade_source"] == "file": - trades = load_backtest_data(Path(config["exportfilename"])) + trades = load_trades(config) pair_counter = 0 for pair, data in tickers.items(): From c3db4ebbc3ed4fff2206d508633c88aabdec06c3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 Jun 2019 20:52:33 +0200 Subject: [PATCH 09/25] Revise plot_profit to use pandas functions where possible --- scripts/plot_profit.py | 145 ++++++++++------------------------------- 1 file changed, 35 insertions(+), 110 deletions(-) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 01dc260d9..5bff9b2dd 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -4,64 +4,28 @@ Script to display profits Use `python plot_profit.py --help` to display the command line arguments """ -import json import logging import sys from argparse import Namespace from pathlib import Path -from typing import List, Optional +from typing import List -import numpy as np +import pandas as pd import plotly.graph_objs as go from plotly import tools from plotly.offline import plot -from freqtrade.arguments import Arguments, ARGS_PLOT_PROFIT +from freqtrade.arguments import ARGS_PLOT_PROFIT, Arguments from freqtrade.configuration import Configuration from freqtrade.data import history -from freqtrade.exchange import timeframe_to_seconds -from freqtrade.misc import common_datearray +from freqtrade.data.btanalysis import create_cum_profit, load_trades +from freqtrade.plot.plotting import generate_plot_file from freqtrade.resolvers import StrategyResolver from freqtrade.state import RunMode - logger = logging.getLogger(__name__) -# data:: [ pair, profit-%, enter, exit, time, duration] -# data:: ["ETH/BTC", 0.0023975, "1515598200", "1515602100", "2018-01-10 07:30:00+00:00", 65] -def make_profit_array(data: List, px: int, min_date: int, - interval: str, - filter_pairs: Optional[List] = None) -> np.ndarray: - pg = np.zeros(px) - filter_pairs = filter_pairs or [] - # Go through the trades - # and make an total profit - # array - for trade in data: - pair = trade[0] - if filter_pairs and pair not in filter_pairs: - continue - profit = trade[1] - trade_sell_time = int(trade[3]) - - ix = define_index(min_date, trade_sell_time, interval) - if ix < px: - logger.debug('[%s]: Add profit %s on %s', pair, profit, trade[4]) - pg[ix] += profit - - # rewrite the pg array to go from - # total profits at each timeframe - # to accumulated profits - pa = 0 - for x in range(0, len(pg)): - p = pg[x] # Get current total percent - pa += p # Add to the accumulated percent - pg[x] = pa # write back to save memory - - return pg - - def plot_profit(args: Namespace) -> None: """ Plots the total profit for all pairs. @@ -70,34 +34,15 @@ def plot_profit(args: Namespace) -> None: in helping out to find a good algorithm. """ - # We need to use the same pairs, same ticker_interval - # and same timeperiod as used in backtesting - # to match the tickerdata against the profits-results + # We need to use the same pairs and the same ticker_interval + # as used in backtesting / trading + # to match the tickerdata against the results timerange = Arguments.parse_timerange(args.timerange) config = Configuration(args, RunMode.OTHER).get_config() # Init strategy - try: - strategy = StrategyResolver({'strategy': config.get('strategy')}).strategy - - except AttributeError: - logger.critical( - 'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"', - config.get('strategy') - ) - exit(1) - - # Load the profits results - try: - filename = args.exportfilename - with open(filename) as file: - data = json.load(file) - except FileNotFoundError: - logger.critical( - 'File "backtest-result.json" not found. This script require backtesting ' - 'results to run.\nPlease run a backtesting with the parameter --export.') - exit(1) + strategy = StrategyResolver(config).strategy # Take pairs from the cli otherwise switch to the pair in the config file if args.pairs: @@ -106,6 +51,11 @@ def plot_profit(args: Namespace) -> None: else: filter_pairs = config['exchange']['pair_whitelist'] + # Load the profits results + trades = load_trades(config) + + trades = trades[trades['pair'].isin(filter_pairs)] + ticker_interval = strategy.ticker_interval pairs = config['exchange']['pair_whitelist'] @@ -120,49 +70,28 @@ def plot_profit(args: Namespace) -> None: refresh_pairs=False, timerange=timerange ) - dataframes = strategy.tickerdata_to_dataframe(tickers) - # NOTE: the dataframes are of unequal length, - # 'dates' is an merged date array of them all. - - dates = common_datearray(dataframes) - min_date = int(min(dates).timestamp()) - max_date = int(max(dates).timestamp()) - num_iterations = define_index(min_date, max_date, ticker_interval) + 1 - - # Make an average close price of all the pairs that was involved. + # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend - # We are essentially saying: - # array <- sum dataframes[*]['close'] / num_items dataframes - # FIX: there should be some onliner numpy/panda for this - avgclose = np.zeros(num_iterations) - num = 0 - for pair, pair_data in dataframes.items(): - close = pair_data['close'] - maxprice = max(close) # Normalize price to [0,1] - logger.info('Pair %s has length %s' % (pair, len(close))) - for x in range(0, len(close)): - avgclose[x] += close[x] / maxprice - # avgclose += close - num += 1 - avgclose /= num - # make an profits-growth array - pg = make_profit_array(data, num_iterations, min_date, ticker_interval, filter_pairs) + # Combine close-values for all pairs, rename columns to "pair" + df_comb = pd.concat([tickers[pair].set_index('date').rename( + {'close': pair}, axis=1)[pair] for pair in tickers], axis=1) + df_comb['mean'] = df_comb.mean(axis=1) + + # Add combined cumulative profit + df_comb = create_cum_profit(df_comb, trades, 'cum_profit') - # # Plot the pairs average close prices, and total profit growth - # - avgclose = go.Scattergl( - x=dates, - y=avgclose, + x=df_comb.index, + y=df_comb['mean'], name='Avg close price', ) profit = go.Scattergl( - x=dates, - y=pg, + x=df_comb.index, + y=df_comb['cum_profit'], name='Profit', ) @@ -172,23 +101,19 @@ def plot_profit(args: Namespace) -> None: fig.append_trace(profit, 2, 1) for pair in pairs: - pg = make_profit_array(data, num_iterations, min_date, ticker_interval, [pair]) + profit_col = f'cum_profit_{pair}' + df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col) + pair_profit = go.Scattergl( - x=dates, - y=pg, - name=pair, + x=df_comb.index, + y=df_comb[profit_col], + name=f"Profit {pair}", ) fig.append_trace(pair_profit, 3, 1) - plot(fig, filename=str(Path('user_data').joinpath('freqtrade-profit-plot.html'))) - - -def define_index(min_date: int, max_date: int, ticker_interval: str) -> int: - """ - Return the index of a specific date - """ - interval_seconds = timeframe_to_seconds(ticker_interval) - return int((max_date - min_date) / interval_seconds) + generate_plot_file(fig, + filename='freqtrade-profit-plot.html', + auto_open=True) def plot_parse_args(args: List[str]) -> Namespace: From 700bab7279bd37d97528123d60b43dfba8b9e882 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 09:28:34 +0200 Subject: [PATCH 10/25] Rename generate_plot_file to store_plot_file --- freqtrade/plot/plotting.py | 2 +- freqtrade/tests/test_plotting.py | 4 ++-- scripts/plot_dataframe.py | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index f8a010068..68d61bb92 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -213,7 +213,7 @@ def generate_plot_filename(pair, ticker_interval) -> str: return file_name -def generate_plot_file(fig, filename: str, auto_open: bool = False) -> None: +def store_plot_file(fig, filename: str, auto_open: bool = False) -> None: """ Generate a plot html file from pre populated fig plotly object :param fig: Plotly Figure to plot diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index 527534522..0e93e8fad 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -9,7 +9,7 @@ from freqtrade.arguments import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import load_backtest_data from freqtrade.plot.plotting import (generate_candlestick_graph, - generate_plot_file, + store_plot_file, generate_plot_filename, generate_row, plot_trades) from freqtrade.strategy.default_strategy import DefaultStrategy @@ -188,7 +188,7 @@ def test_generate_Plot_filename(): def test_generate_plot_file(mocker, caplog): fig = generage_empty_figure() plot_mock = mocker.patch("freqtrade.plot.plotting.plot", MagicMock()) - generate_plot_file(fig, filename="freqtrade-plot-UNITTEST_BTC-5m.html") + store_plot_file(fig, filename="freqtrade-plot-UNITTEST_BTC-5m.html") assert plot_mock.call_count == 1 assert plot_mock.call_args[0][0] == fig diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index dfd31e9ab..a0f4ef778 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -24,7 +24,7 @@ from freqtrade.data import history from freqtrade.data.btanalysis import extract_trades_of_period, load_trades from freqtrade.optimize import setup_configuration from freqtrade.plot.plotting import (generate_candlestick_graph, - generate_plot_file, + store_plot_file, generate_plot_filename) from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode @@ -67,7 +67,6 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): # Set timerange to use timerange = Arguments.parse_timerange(config["timerange"]) - ticker_interval = strategy.ticker_interval tickers = history.load_data( datadir=Path(str(config.get("datadir"))), @@ -100,7 +99,7 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): indicators2=config["indicators2"].split(",") ) - generate_plot_file(fig, generate_plot_filename(pair, ticker_interval)) + store_plot_file(fig, generate_plot_filename(pair, config['ticker_interval'])) logger.info('End of ploting process %s plots generated', pair_counter) From c87d27048bbd8e5c53bec3d49f0336dec1953dfd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 09:28:49 +0200 Subject: [PATCH 11/25] align plot_profit to plot_dataframe --- scripts/plot_profit.py | 69 +++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 38 deletions(-) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 5bff9b2dd..f28763077 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -6,27 +6,25 @@ Use `python plot_profit.py --help` to display the command line arguments """ import logging import sys -from argparse import Namespace from pathlib import Path -from typing import List +from typing import Any, Dict, List import pandas as pd import plotly.graph_objs as go from plotly import tools -from plotly.offline import plot from freqtrade.arguments import ARGS_PLOT_PROFIT, Arguments -from freqtrade.configuration import Configuration from freqtrade.data import history from freqtrade.data.btanalysis import create_cum_profit, load_trades -from freqtrade.plot.plotting import generate_plot_file -from freqtrade.resolvers import StrategyResolver +from freqtrade.optimize import setup_configuration +from freqtrade.plot.plotting import store_plot_file +from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode logger = logging.getLogger(__name__) -def plot_profit(args: Namespace) -> None: +def plot_profit(config: Dict[str, Any]) -> None: """ Plots the total profit for all pairs. Note, the profit calculation isn't realistic. @@ -34,42 +32,33 @@ def plot_profit(args: Namespace) -> None: in helping out to find a good algorithm. """ + exchange = ExchangeResolver(config.get('exchange', {}).get('name'), config).exchange + + # Take pairs from the cli otherwise switch to the pair in the config file + if "pairs" in config: + pairs = config["pairs"].split(',') + else: + pairs = config["exchange"]["pair_whitelist"] + # We need to use the same pairs and the same ticker_interval # as used in backtesting / trading # to match the tickerdata against the results - timerange = Arguments.parse_timerange(args.timerange) + timerange = Arguments.parse_timerange(config["timerange"]) - config = Configuration(args, RunMode.OTHER).get_config() - - # Init strategy - strategy = StrategyResolver(config).strategy - - # Take pairs from the cli otherwise switch to the pair in the config file - if args.pairs: - filter_pairs = args.pairs - filter_pairs = filter_pairs.split(',') - else: - filter_pairs = config['exchange']['pair_whitelist'] + tickers = history.load_data( + datadir=Path(str(config.get("datadir"))), + pairs=pairs, + ticker_interval=config['ticker_interval'], + refresh_pairs=config.get('refresh_pairs', False), + timerange=timerange, + exchange=exchange, + live=config.get("live", False), + ) # Load the profits results trades = load_trades(config) - trades = trades[trades['pair'].isin(filter_pairs)] - - ticker_interval = strategy.ticker_interval - pairs = config['exchange']['pair_whitelist'] - - if filter_pairs: - pairs = list(set(pairs) & set(filter_pairs)) - logger.info('Filter, keep pairs %s' % pairs) - - tickers = history.load_data( - datadir=Path(str(config.get('datadir'))), - pairs=pairs, - ticker_interval=ticker_interval, - refresh_pairs=False, - timerange=timerange - ) + trades = trades[trades['pair'].isin(pairs)] # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend @@ -111,12 +100,12 @@ def plot_profit(args: Namespace) -> None: ) fig.append_trace(pair_profit, 3, 1) - generate_plot_file(fig, + store_plot_file(fig, filename='freqtrade-profit-plot.html', auto_open=True) -def plot_parse_args(args: List[str]) -> Namespace: +def plot_parse_args(args: List[str]) -> Dict[str, Any]: """ Parse args passed to the script :param args: Cli arguments @@ -125,7 +114,11 @@ def plot_parse_args(args: List[str]) -> Namespace: arguments = Arguments(args, 'Graph profits') arguments.build_args(optionlist=ARGS_PLOT_PROFIT) - return arguments.parse_args() + parsed_args = arguments.parse_args() + + # Load the configuration + config = setup_configuration(parsed_args, RunMode.OTHER) + return config def main(sysargv: List[str]) -> None: From 42ea0a19d2b79e319398569aa543c84ac46c3e92 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 09:41:43 +0200 Subject: [PATCH 12/25] create FTPlots class to combine duplicate script code --- freqtrade/plot/plotting.py | 42 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 68d61bb92..868ffbc31 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -1,8 +1,14 @@ import logging -from typing import List +from pathlib import Path +from typing import Any, Dict, List, Optional import pandas as pd -from pathlib import Path + +from freqtrade.arguments import Arguments +from frqtrade.exchange import Exchange +from freqtrade.data import history +from freqtrade.data.btanalysis import load_trades +from freqtrade.resolvers import ExchangeResolver, StrategyResolver logger = logging.getLogger(__name__) @@ -16,6 +22,38 @@ except ImportError: exit(1) +class FTPlots(): + + def __init__(self, config: Dict[str, Any]): + self._config = config + self.exchange: Optional[Exchange] = None + + if self._config.get("live", False) or self._config.get("refresh_pairs", False): + self.exchange = ExchangeResolver(self._config.get('exchange', {}).get('name'), + self._config).exchange + + self.strategy = StrategyResolver(self._config).strategy + if "pairs" in self._config: + self.pairs = self._config["pairs"].split(',') + else: + self.pairs = self._config["exchange"]["pair_whitelist"] + + # Set timerange to use + self.timerange = Arguments.parse_timerange(self._config["timerange"]) + + self.tickers = history.load_data( + datadir=Path(str(self._config.get("datadir"))), + pairs=self.pairs, + ticker_interval=self._config['ticker_interval'], + refresh_pairs=self._config.get('refresh_pairs', False), + timerange=self.timerange, + exchange=self.exchange, + live=self._config.get("live", False), + ) + + self.trades = load_trades(self._config) + + def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots: """ Generator all the indicator selected by the user for a specific row From 88545d882ca7df3b52586b0058ad9bf94fc0fe2e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 09:42:10 +0200 Subject: [PATCH 13/25] Use FTPlots class in plot-scripts --- scripts/plot_dataframe.py | 36 +++++------------------------- scripts/plot_profit.py | 46 +++++++-------------------------------- 2 files changed, 14 insertions(+), 68 deletions(-) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index a0f4ef778..431c6239c 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -14,19 +14,16 @@ Example of usage: """ import logging import sys -from pathlib import Path from typing import Any, Dict, List import pandas as pd from freqtrade.arguments import ARGS_PLOT_DATAFRAME, Arguments -from freqtrade.data import history -from freqtrade.data.btanalysis import extract_trades_of_period, load_trades +from freqtrade.data.btanalysis import extract_trades_of_period from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import (generate_candlestick_graph, +from freqtrade.plot.plotting import (FTPlots, generate_candlestick_graph, store_plot_file, generate_plot_filename) -from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -57,38 +54,17 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): -Generate plot files :return: None """ - exchange = ExchangeResolver(config.get('exchange', {}).get('name'), config).exchange - - strategy = StrategyResolver(config).strategy - if "pairs" in config: - pairs = config["pairs"].split(',') - else: - pairs = config["exchange"]["pair_whitelist"] - - # Set timerange to use - timerange = Arguments.parse_timerange(config["timerange"]) - - tickers = history.load_data( - datadir=Path(str(config.get("datadir"))), - pairs=pairs, - ticker_interval=config['ticker_interval'], - refresh_pairs=config.get('refresh_pairs', False), - timerange=timerange, - exchange=exchange, - live=config.get("live", False), - ) - - trades = load_trades(config) + plot = FTPlots(config) pair_counter = 0 - for pair, data in tickers.items(): + for pair, data in plot.tickers.items(): pair_counter += 1 logger.info("analyse pair %s", pair) tickers = {} tickers[pair] = data - dataframe = generate_dataframe(strategy, tickers, pair) + dataframe = generate_dataframe(plot.strategy, tickers, pair) - trades_pair = trades.loc[trades['pair'] == pair] + trades_pair = plot.trades.loc[plot.trades['pair'] == pair] trades_pair = extract_trades_of_period(dataframe, trades_pair) fig = generate_candlestick_graph( diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index f28763077..cd507100f 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -6,7 +6,6 @@ Use `python plot_profit.py --help` to display the command line arguments """ import logging import sys -from pathlib import Path from typing import Any, Dict, List import pandas as pd @@ -14,11 +13,9 @@ import plotly.graph_objs as go from plotly import tools from freqtrade.arguments import ARGS_PLOT_PROFIT, Arguments -from freqtrade.data import history -from freqtrade.data.btanalysis import create_cum_profit, load_trades +from freqtrade.data.btanalysis import create_cum_profit from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import store_plot_file -from freqtrade.resolvers import ExchangeResolver +from freqtrade.plot.plotting import FTPlots, store_plot_file from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -31,41 +28,16 @@ def plot_profit(config: Dict[str, Any]) -> None: But should be somewhat proportional, and therefor useful in helping out to find a good algorithm. """ + plot = FTPlots(config) - exchange = ExchangeResolver(config.get('exchange', {}).get('name'), config).exchange - - # Take pairs from the cli otherwise switch to the pair in the config file - if "pairs" in config: - pairs = config["pairs"].split(',') - else: - pairs = config["exchange"]["pair_whitelist"] - - # We need to use the same pairs and the same ticker_interval - # as used in backtesting / trading - # to match the tickerdata against the results - timerange = Arguments.parse_timerange(config["timerange"]) - - tickers = history.load_data( - datadir=Path(str(config.get("datadir"))), - pairs=pairs, - ticker_interval=config['ticker_interval'], - refresh_pairs=config.get('refresh_pairs', False), - timerange=timerange, - exchange=exchange, - live=config.get("live", False), - ) - - # Load the profits results - trades = load_trades(config) - - trades = trades[trades['pair'].isin(pairs)] + trades = plot.trades[plot.trades['pair'].isin(plot.pairs)] # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend # Combine close-values for all pairs, rename columns to "pair" - df_comb = pd.concat([tickers[pair].set_index('date').rename( - {'close': pair}, axis=1)[pair] for pair in tickers], axis=1) + df_comb = pd.concat([plot.tickers[pair].set_index('date').rename( + {'close': pair}, axis=1)[pair] for pair in plot.tickers], axis=1) df_comb['mean'] = df_comb.mean(axis=1) # Add combined cumulative profit @@ -89,7 +61,7 @@ def plot_profit(config: Dict[str, Any]) -> None: fig.append_trace(avgclose, 1, 1) fig.append_trace(profit, 2, 1) - for pair in pairs: + for pair in plot.pairs: profit_col = f'cum_profit_{pair}' df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col) @@ -100,9 +72,7 @@ def plot_profit(config: Dict[str, Any]) -> None: ) fig.append_trace(pair_profit, 3, 1) - store_plot_file(fig, - filename='freqtrade-profit-plot.html', - auto_open=True) + store_plot_file(fig, filename='freqtrade-profit-plot.html', auto_open=True) def plot_parse_args(args: List[str]) -> Dict[str, Any]: From 0d5e94b147b74a201914b2558e69a5cf2470bd9f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 09:44:50 +0200 Subject: [PATCH 14/25] Rename generate_row to add_indicators --- freqtrade/plot/plotting.py | 6 +++--- freqtrade/tests/test_plotting.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 868ffbc31..8ac4800bb 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -54,7 +54,7 @@ class FTPlots(): self.trades = load_trades(self._config) -def generate_row(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots: +def add_indicators(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots: """ Generator all the indicator selected by the user for a specific row :param fig: Plot figure to append to @@ -224,7 +224,7 @@ def generate_candlestick_graph( fig.append_trace(bb_upper, 1, 1) # Add indicators to main plot - fig = generate_row(fig=fig, row=1, indicators=indicators1, data=data) + fig = add_indicators(fig=fig, row=1, indicators=indicators1, data=data) fig = plot_trades(fig, trades) @@ -237,7 +237,7 @@ def generate_candlestick_graph( fig.append_trace(volume, 2, 1) # Add indicators to seperate row - fig = generate_row(fig=fig, row=3, indicators=indicators2, data=data) + fig = add_indicators(fig=fig, row=3, indicators=indicators2, data=data) return fig diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index 0e93e8fad..e4d913f70 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -10,14 +10,14 @@ from freqtrade.data import history from freqtrade.data.btanalysis import load_backtest_data from freqtrade.plot.plotting import (generate_candlestick_graph, store_plot_file, - generate_plot_filename, generate_row, + generate_plot_filename, add_indicators, plot_trades) from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.tests.conftest import log_has, log_has_re def fig_generating_mock(fig, *args, **kwargs): - """ Return Fig - used to mock generate_row and plot_trades""" + """ Return Fig - used to mock add_indicators and plot_trades""" return fig @@ -36,7 +36,7 @@ def generage_empty_figure(): ) -def test_generate_row(default_conf, caplog): +def test_add_indicators(default_conf, caplog): pair = "UNITTEST/BTC" timerange = TimeRange(None, 'line', 0, -1000) @@ -51,20 +51,20 @@ def test_generate_row(default_conf, caplog): fig = generage_empty_figure() # Row 1 - fig1 = generate_row(fig=deepcopy(fig), row=1, indicators=indicators1, data=data) + fig1 = add_indicators(fig=deepcopy(fig), row=1, indicators=indicators1, data=data) figure = fig1.layout.figure ema10 = find_trace_in_fig_data(figure.data, "ema10") assert isinstance(ema10, go.Scatter) assert ema10.yaxis == "y" - fig2 = generate_row(fig=deepcopy(fig), row=3, indicators=indicators2, data=data) + fig2 = add_indicators(fig=deepcopy(fig), row=3, indicators=indicators2, data=data) figure = fig2.layout.figure macd = find_trace_in_fig_data(figure.data, "macd") assert isinstance(macd, go.Scatter) assert macd.yaxis == "y3" # No indicator found - fig3 = generate_row(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data) + fig3 = add_indicators(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data) assert fig == fig3 assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog.record_tuples) @@ -98,7 +98,7 @@ def test_plot_trades(caplog): def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, caplog): - row_mock = mocker.patch('freqtrade.plot.plotting.generate_row', + row_mock = mocker.patch('freqtrade.plot.plotting.add_indicators', MagicMock(side_effect=fig_generating_mock)) trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades', MagicMock(side_effect=fig_generating_mock)) @@ -134,7 +134,7 @@ def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, c def test_generate_candlestick_graph_no_trades(default_conf, mocker): - row_mock = mocker.patch('freqtrade.plot.plotting.generate_row', + row_mock = mocker.patch('freqtrade.plot.plotting.add_indicators', MagicMock(side_effect=fig_generating_mock)) trades_mock = mocker.patch('freqtrade.plot.plotting.plot_trades', MagicMock(side_effect=fig_generating_mock)) From 348513c1514311894035642591d046d035c73df3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 09:47:07 +0200 Subject: [PATCH 15/25] Improve formatting of plotting.py --- freqtrade/plot/plotting.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 8ac4800bb..2fe15d320 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -84,7 +84,7 @@ def add_indicators(fig, row, indicators: List[str], data: pd.DataFrame) -> tools def plot_trades(fig, trades: pd.DataFrame): """ - Plot trades to "fig" + Add trades to "fig" """ # Trades can be empty if trades is not None and len(trades) > 0: @@ -124,13 +124,9 @@ def plot_trades(fig, trades: pd.DataFrame): return fig -def generate_candlestick_graph( - pair: str, - data: pd.DataFrame, - trades: pd.DataFrame = None, - indicators1: List[str] = [], - indicators2: List[str] = [], -) -> go.Figure: +def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFrame = None, + indicators1: List[str] = [], + indicators2: List[str] = [],) -> go.Figure: """ Generate the graph from the data generated by Backtesting or from DB Volume will always be ploted in row2, so Row 1 and 3 are to our disposal for custom indicators @@ -243,6 +239,9 @@ def generate_candlestick_graph( def generate_plot_filename(pair, ticker_interval) -> str: + """ + Generate filenames per pair/ticker_interval to be used for storing plots + """ pair_name = pair.replace("/", "_") file_name = 'freqtrade-plot-' + pair_name + '-' + ticker_interval + '.html' From 6b387d320e594bd8c82bd91cca951a6a09dc3050 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 10:04:43 +0200 Subject: [PATCH 16/25] extract combine_tickers to btanalysis --- freqtrade/data/btanalysis.py | 19 ++++++++++++++++++- freqtrade/tests/data/test_btanalysis.py | 20 ++++++++++++++++++-- scripts/plot_profit.py | 6 ++---- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index f6f7e5541..834d41263 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -3,6 +3,7 @@ Helpers when analyzing backtest data """ import logging from pathlib import Path +from typing import Dict import numpy as np import pandas as pd @@ -125,7 +126,23 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> p return trades -def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str): +def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame], column: str = "close"): + """ + Combine multiple dataframes "column" + :param tickers: Dict of Dataframes, dict key should be pair. + :param column: Column in the original dataframes to use + :return: DataFrame with the column renamed to the dict key, and a column + named mean, containing the mean of all pairs. + """ + df_comb = pd.concat([tickers[pair].set_index('date').rename( + {column: pair}, axis=1)[pair] for pair in tickers], axis=1) + + df_comb['mean'] = df_comb.mean(axis=1) + + return df_comb + + +def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str) -> pd.DataFrame: """ Adds a column `col_name` with the cumulative profit for the given trades array. :param df: DataFrame with date index diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index 01e5dc90d..e8872f9a4 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -5,11 +5,14 @@ from arrow import Arrow from pandas import DataFrame, to_datetime from freqtrade.arguments import Arguments, TimeRange -from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, create_cum_profit, +from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, + combine_tickers_with_mean, + create_cum_profit, extract_trades_of_period, load_backtest_data, load_trades, load_trades_from_db) -from freqtrade.data.history import load_pair_history, make_testdata_path +from freqtrade.data.history import (load_data, load_pair_history, + make_testdata_path) from freqtrade.tests.test_persistence import create_mock_trades @@ -97,6 +100,19 @@ def test_load_trades(default_conf, mocker): assert bt_mock.call_count == 1 +def test_combine_tickers_with_mean(): + pairs = ["ETH/BTC", "XLM/BTC"] + tickers = load_data(datadir=None, + pairs=pairs, + ticker_interval='5m' + ) + df = combine_tickers_with_mean(tickers) + assert isinstance(df, DataFrame) + assert "ETH/BTC" in df.columns + assert "XLM/BTC" in df.columns + assert "mean" in df.columns + + def test_create_cum_profit(): filename = make_testdata_path(None) / "backtest-result_test.json" bt_data = load_backtest_data(filename) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index cd507100f..248eeb7b0 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -13,7 +13,7 @@ import plotly.graph_objs as go from plotly import tools from freqtrade.arguments import ARGS_PLOT_PROFIT, Arguments -from freqtrade.data.btanalysis import create_cum_profit +from freqtrade.data.btanalysis import create_cum_profit, combine_tickers_with_mean from freqtrade.optimize import setup_configuration from freqtrade.plot.plotting import FTPlots, store_plot_file from freqtrade.state import RunMode @@ -36,9 +36,7 @@ def plot_profit(config: Dict[str, Any]) -> None: # this could be useful to gauge the overall market trend # Combine close-values for all pairs, rename columns to "pair" - df_comb = pd.concat([plot.tickers[pair].set_index('date').rename( - {'close': pair}, axis=1)[pair] for pair in plot.tickers], axis=1) - df_comb['mean'] = df_comb.mean(axis=1) + df_comb = combine_tickers_with_mean(plot.tickers, "close") # Add combined cumulative profit df_comb = create_cum_profit(df_comb, trades, 'cum_profit') From 0a184d380e0c0c045f7a4539da100fade9db51a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 10:14:33 +0200 Subject: [PATCH 17/25] create add_profit function --- freqtrade/plot/plotting.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 2fe15d320..f8736fe74 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -5,9 +5,9 @@ from typing import Any, Dict, List, Optional import pandas as pd from freqtrade.arguments import Arguments -from frqtrade.exchange import Exchange +from freqtrade.exchange import Exchange from freqtrade.data import history -from freqtrade.data.btanalysis import load_trades +from freqtrade.data.btanalysis import load_trades, create_cum_profit from freqtrade.resolvers import ExchangeResolver, StrategyResolver logger = logging.getLogger(__name__) @@ -82,7 +82,27 @@ def add_indicators(fig, row, indicators: List[str], data: pd.DataFrame) -> tools return fig -def plot_trades(fig, trades: pd.DataFrame): +def add_profit(fig, row, data: pd.DataFrame, column: str, name: str) -> tools.make_subplots: + """ + Add profit-plot + :param fig: Plot figure to append to + :param row: row number for this plot + :param data: candlestick DataFrame + :param column: Column to use for plot + :param name: Name to use + :return: fig with added profit plot + """ + profit = go.Scattergl( + x=data.index, + y=data[column], + name=name, + ) + fig.append_trace(profit, row, 1) + + return fig + + +def plot_trades(fig, trades: pd.DataFrame) -> tools.make_subplots: """ Add trades to "fig" """ From 5a11ffcad808ede0990c72356e8e19c627aca0e3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 10:24:10 +0200 Subject: [PATCH 18/25] Add test for add_profit --- freqtrade/data/btanalysis.py | 2 -- freqtrade/tests/test_plotting.py | 26 +++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 834d41263..556666f4e 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -112,8 +112,6 @@ def load_trades(config) -> pd.DataFrame: return load_trades_from_db(config["db_url"]) elif config["trade_source"] == "file": return load_backtest_data(Path(config["exportfilename"])) - else: - return None def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame: diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index e4d913f70..2a73ad24e 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -5,11 +5,11 @@ from unittest.mock import MagicMock import plotly.graph_objs as go from plotly import tools -from freqtrade.arguments import TimeRange +from freqtrade.arguments import TimeRange, Arguments from freqtrade.data import history -from freqtrade.data.btanalysis import load_backtest_data +from freqtrade.data.btanalysis import load_backtest_data, create_cum_profit from freqtrade.plot.plotting import (generate_candlestick_graph, - store_plot_file, + store_plot_file, add_profit, generate_plot_filename, add_indicators, plot_trades) from freqtrade.strategy.default_strategy import DefaultStrategy @@ -194,3 +194,23 @@ def test_generate_plot_file(mocker, caplog): assert plot_mock.call_args[0][0] == fig assert (plot_mock.call_args_list[0][1]['filename'] == "user_data/plots/freqtrade-plot-UNITTEST_BTC-5m.html") + + +def test_add_profit(): + filename = history.make_testdata_path(None) / "backtest-result_test.json" + bt_data = load_backtest_data(filename) + timerange = Arguments.parse_timerange("20180110-20180112") + + df = history.load_pair_history(pair="POWR/BTC", ticker_interval='5m', + datadir=None, timerange=timerange) + fig = generage_empty_figure() + + cum_profits = create_cum_profit(df.set_index('date'), + bt_data[bt_data["pair"] == 'POWR/BTC'], + "cum_profits") + + fig1 = add_profit(fig, row=2, data=cum_profits, column='cum_profits', name='Profits') + figure = fig1.layout.figure + profits = find_trace_in_fig_data(figure.data, "Profits") + assert isinstance(profits, go.Scattergl) + assert profits.yaxis == "y2" From 0b517584aacc12d9c1b054b77c29a52633c9b3b8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 10:25:48 +0200 Subject: [PATCH 19/25] Use add_profit in script --- freqtrade/plot/plotting.py | 2 +- freqtrade/tests/test_plotting.py | 12 ++++++------ scripts/plot_profit.py | 18 +++--------------- 3 files changed, 10 insertions(+), 22 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index f8736fe74..4c45c0375 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -7,7 +7,7 @@ import pandas as pd from freqtrade.arguments import Arguments from freqtrade.exchange import Exchange from freqtrade.data import history -from freqtrade.data.btanalysis import load_trades, create_cum_profit +from freqtrade.data.btanalysis import load_trades from freqtrade.resolvers import ExchangeResolver, StrategyResolver logger = logging.getLogger(__name__) diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index 2a73ad24e..fb2c52e1e 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -5,13 +5,13 @@ from unittest.mock import MagicMock import plotly.graph_objs as go from plotly import tools -from freqtrade.arguments import TimeRange, Arguments +from freqtrade.arguments import Arguments, TimeRange from freqtrade.data import history -from freqtrade.data.btanalysis import load_backtest_data, create_cum_profit -from freqtrade.plot.plotting import (generate_candlestick_graph, - store_plot_file, add_profit, - generate_plot_filename, add_indicators, - plot_trades) +from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data +from freqtrade.plot.plotting import (add_indicators, add_profit, + generate_candlestick_graph, + generate_plot_filename, plot_trades, + store_plot_file) from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.tests.conftest import log_has, log_has_re diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index 248eeb7b0..ad135b5e6 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -8,14 +8,13 @@ import logging import sys from typing import Any, Dict, List -import pandas as pd import plotly.graph_objs as go from plotly import tools from freqtrade.arguments import ARGS_PLOT_PROFIT, Arguments from freqtrade.data.btanalysis import create_cum_profit, combine_tickers_with_mean from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import FTPlots, store_plot_file +from freqtrade.plot.plotting import FTPlots, store_plot_file, add_profit from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -48,27 +47,16 @@ def plot_profit(config: Dict[str, Any]) -> None: name='Avg close price', ) - profit = go.Scattergl( - x=df_comb.index, - y=df_comb['cum_profit'], - name='Profit', - ) - fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1]) fig.append_trace(avgclose, 1, 1) - fig.append_trace(profit, 2, 1) + fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit') for pair in plot.pairs: profit_col = f'cum_profit_{pair}' df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col) - pair_profit = go.Scattergl( - x=df_comb.index, - y=df_comb[profit_col], - name=f"Profit {pair}", - ) - fig.append_trace(pair_profit, 3, 1) + fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}") store_plot_file(fig, filename='freqtrade-profit-plot.html', auto_open=True) From c7a4a16eec3778ccc6b8b2cdab66e0c3434ad242 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 10:31:36 +0200 Subject: [PATCH 20/25] Create generate_plot_graph --- freqtrade/plot/plotting.py | 35 +++++++++++++++++++++++++++++++++-- scripts/plot_profit.py | 33 ++------------------------------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 4c45c0375..04e246371 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -5,9 +5,10 @@ from typing import Any, Dict, List, Optional import pandas as pd from freqtrade.arguments import Arguments -from freqtrade.exchange import Exchange from freqtrade.data import history -from freqtrade.data.btanalysis import load_trades +from freqtrade.data.btanalysis import (combine_tickers_with_mean, + create_cum_profit, load_trades) +from freqtrade.exchange import Exchange from freqtrade.resolvers import ExchangeResolver, StrategyResolver logger = logging.getLogger(__name__) @@ -28,6 +29,7 @@ class FTPlots(): self._config = config self.exchange: Optional[Exchange] = None + # Exchange is only needed when downloading data! if self._config.get("live", False) or self._config.get("refresh_pairs", False): self.exchange = ExchangeResolver(self._config.get('exchange', {}).get('name'), self._config).exchange @@ -258,6 +260,35 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra return fig +def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], trades: pd.DataFrame = None, + ) -> go.Figure: + # Combine close-values for all pairs, rename columns to "pair" + df_comb = combine_tickers_with_mean(tickers, "close") + + # Add combined cumulative profit + df_comb = create_cum_profit(df_comb, trades, 'cum_profit') + + # Plot the pairs average close prices, and total profit growth + avgclose = go.Scattergl( + x=df_comb.index, + y=df_comb['mean'], + name='Avg close price', + ) + + fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1]) + + fig.append_trace(avgclose, 1, 1) + fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit') + + for pair in pairs: + profit_col = f'cum_profit_{pair}' + df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col) + + fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}") + + store_plot_file(fig, filename='freqtrade-profit-plot.html', auto_open=True) + + def generate_plot_filename(pair, ticker_interval) -> str: """ Generate filenames per pair/ticker_interval to be used for storing plots diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index ad135b5e6..f1cf99828 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -8,13 +8,9 @@ import logging import sys from typing import Any, Dict, List -import plotly.graph_objs as go -from plotly import tools - from freqtrade.arguments import ARGS_PLOT_PROFIT, Arguments -from freqtrade.data.btanalysis import create_cum_profit, combine_tickers_with_mean from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import FTPlots, store_plot_file, add_profit +from freqtrade.plot.plotting import FTPlots, generate_profit_graph from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -33,32 +29,7 @@ def plot_profit(config: Dict[str, Any]) -> None: # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend - - # Combine close-values for all pairs, rename columns to "pair" - df_comb = combine_tickers_with_mean(plot.tickers, "close") - - # Add combined cumulative profit - df_comb = create_cum_profit(df_comb, trades, 'cum_profit') - - # Plot the pairs average close prices, and total profit growth - avgclose = go.Scattergl( - x=df_comb.index, - y=df_comb['mean'], - name='Avg close price', - ) - - fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1]) - - fig.append_trace(avgclose, 1, 1) - fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit') - - for pair in plot.pairs: - profit_col = f'cum_profit_{pair}' - df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col) - - fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}") - - store_plot_file(fig, filename='freqtrade-profit-plot.html', auto_open=True) + generate_profit_graph(plot.pairs, plot.tickers, trades) def plot_parse_args(args: List[str]) -> Dict[str, Any]: From 587d71efb586b336e5c7ed50feb83b5f1f70afbd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 10:47:55 +0200 Subject: [PATCH 21/25] Test generate_profit_plot --- freqtrade/plot/plotting.py | 3 ++- freqtrade/tests/test_plotting.py | 34 +++++++++++++++++++++++++++++++- scripts/plot_profit.py | 5 +++-- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 04e246371..922f2847f 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -276,6 +276,7 @@ def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], trades: ) fig = tools.make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1]) + fig['layout'].update(title="Profit plot") fig.append_trace(avgclose, 1, 1) fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit') @@ -286,7 +287,7 @@ def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], trades: fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}") - store_plot_file(fig, filename='freqtrade-profit-plot.html', auto_open=True) + return fig def generate_plot_filename(pair, ticker_interval) -> str: diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index fb2c52e1e..32e7dcd8a 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -10,7 +10,8 @@ from freqtrade.data import history from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data from freqtrade.plot.plotting import (add_indicators, add_profit, generate_candlestick_graph, - generate_plot_filename, plot_trades, + generate_plot_filename, + generate_profit_graph, plot_trades, store_plot_file) from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.tests.conftest import log_has, log_has_re @@ -214,3 +215,34 @@ def test_add_profit(): profits = find_trace_in_fig_data(figure.data, "Profits") assert isinstance(profits, go.Scattergl) assert profits.yaxis == "y2" + + +def test_generate_profit_graph(): + filename = history.make_testdata_path(None) / "backtest-result_test.json" + trades = load_backtest_data(filename) + timerange = Arguments.parse_timerange("20180110-20180112") + pairs = ["POWR/BTC", "XLM/BTC"] + + tickers = history.load_data(datadir=None, + pairs=pairs, + ticker_interval='5m', + timerange=timerange + ) + trades = trades[trades['pair'].isin(pairs)] + + fig = generate_profit_graph(pairs, tickers, trades) + assert isinstance(fig, go.Figure) + + assert fig.layout.title.text == "Profit plot" + figure = fig.layout.figure + assert len(figure.data) == 4 + + avgclose = find_trace_in_fig_data(figure.data, "Avg close price") + assert isinstance(avgclose, go.Scattergl) + + profit = find_trace_in_fig_data(figure.data, "Profit") + assert isinstance(profit, go.Scattergl) + + for pair in pairs: + profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}") + assert isinstance(profit_pair, go.Scattergl) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index f1cf99828..c29b4d967 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -10,7 +10,7 @@ from typing import Any, Dict, List from freqtrade.arguments import ARGS_PLOT_PROFIT, Arguments from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import FTPlots, generate_profit_graph +from freqtrade.plot.plotting import FTPlots, generate_profit_graph, store_plot_file from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -29,7 +29,8 @@ def plot_profit(config: Dict[str, Any]) -> None: # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend - generate_profit_graph(plot.pairs, plot.tickers, trades) + fig = generate_profit_graph(plot.pairs, plot.tickers, trades) + store_plot_file(fig, filename='freqtrade-profit-plot.html', auto_open=True) def plot_parse_args(args: List[str]) -> Dict[str, Any]: From db59d39e2c40a96c54e2e1abe5179765a98d3883 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 11:06:51 +0200 Subject: [PATCH 22/25] Don't use class for plotting This will allow easy usage of the methods from jupter notebooks --- freqtrade/plot/plotting.py | 60 +++++++++++++++++++++----------------- scripts/plot_dataframe.py | 11 +++---- scripts/plot_profit.py | 11 +++---- 3 files changed, 45 insertions(+), 37 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 922f2847f..079a098dc 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Dict, List, Optional import pandas as pd @@ -23,37 +23,43 @@ except ImportError: exit(1) -class FTPlots(): +def init_plotscript(config): + """ + Initialize objects needed for plotting + :return: Dict with tickers, trades, pairs and strategy + """ + exchange: Optional[Exchange] = None - def __init__(self, config: Dict[str, Any]): - self._config = config - self.exchange: Optional[Exchange] = None + # Exchange is only needed when downloading data! + if config.get("live", False) or config.get("refresh_pairs", False): + exchange = ExchangeResolver(config.get('exchange', {}).get('name'), + config).exchange - # Exchange is only needed when downloading data! - if self._config.get("live", False) or self._config.get("refresh_pairs", False): - self.exchange = ExchangeResolver(self._config.get('exchange', {}).get('name'), - self._config).exchange + strategy = StrategyResolver(config).strategy + if "pairs" in config: + pairs = config["pairs"].split(',') + else: + pairs = config["exchange"]["pair_whitelist"] - self.strategy = StrategyResolver(self._config).strategy - if "pairs" in self._config: - self.pairs = self._config["pairs"].split(',') - else: - self.pairs = self._config["exchange"]["pair_whitelist"] + # Set timerange to use + timerange = Arguments.parse_timerange(config["timerange"]) - # Set timerange to use - self.timerange = Arguments.parse_timerange(self._config["timerange"]) + tickers = history.load_data( + datadir=Path(str(config.get("datadir"))), + pairs=pairs, + ticker_interval=config['ticker_interval'], + refresh_pairs=config.get('refresh_pairs', False), + timerange=timerange, + exchange=exchange, + live=config.get("live", False), + ) - self.tickers = history.load_data( - datadir=Path(str(self._config.get("datadir"))), - pairs=self.pairs, - ticker_interval=self._config['ticker_interval'], - refresh_pairs=self._config.get('refresh_pairs', False), - timerange=self.timerange, - exchange=self.exchange, - live=self._config.get("live", False), - ) - - self.trades = load_trades(self._config) + trades = load_trades(config) + return {"tickers": tickers, + "trades": trades, + "pairs": pairs, + "strategy": strategy, + } def add_indicators(fig, row, indicators: List[str], data: pd.DataFrame) -> tools.make_subplots: diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py index 431c6239c..1e2d9f248 100755 --- a/scripts/plot_dataframe.py +++ b/scripts/plot_dataframe.py @@ -21,7 +21,7 @@ import pandas as pd from freqtrade.arguments import ARGS_PLOT_DATAFRAME, Arguments from freqtrade.data.btanalysis import extract_trades_of_period from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import (FTPlots, generate_candlestick_graph, +from freqtrade.plot.plotting import (init_plotscript, generate_candlestick_graph, store_plot_file, generate_plot_filename) from freqtrade.state import RunMode @@ -54,17 +54,18 @@ def analyse_and_plot_pairs(config: Dict[str, Any]): -Generate plot files :return: None """ - plot = FTPlots(config) + plot_elements = init_plotscript(config) + trades = plot_elements['trades'] pair_counter = 0 - for pair, data in plot.tickers.items(): + for pair, data in plot_elements["tickers"].items(): pair_counter += 1 logger.info("analyse pair %s", pair) tickers = {} tickers[pair] = data - dataframe = generate_dataframe(plot.strategy, tickers, pair) + dataframe = generate_dataframe(plot_elements["strategy"], tickers, pair) - trades_pair = plot.trades.loc[plot.trades['pair'] == pair] + trades_pair = trades.loc[trades['pair'] == pair] trades_pair = extract_trades_of_period(dataframe, trades_pair) fig = generate_candlestick_graph( diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py index c29b4d967..7442ef155 100755 --- a/scripts/plot_profit.py +++ b/scripts/plot_profit.py @@ -10,7 +10,7 @@ from typing import Any, Dict, List from freqtrade.arguments import ARGS_PLOT_PROFIT, Arguments from freqtrade.optimize import setup_configuration -from freqtrade.plot.plotting import FTPlots, generate_profit_graph, store_plot_file +from freqtrade.plot.plotting import init_plotscript, generate_profit_graph, store_plot_file from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -23,13 +23,14 @@ def plot_profit(config: Dict[str, Any]) -> None: But should be somewhat proportional, and therefor useful in helping out to find a good algorithm. """ - plot = FTPlots(config) - - trades = plot.trades[plot.trades['pair'].isin(plot.pairs)] + plot_elements = init_plotscript(config) + trades = plot_elements['trades'] + # Filter trades to relevant pairs + trades = trades[trades['pair'].isin(plot_elements["pairs"])] # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend - fig = generate_profit_graph(plot.pairs, plot.tickers, trades) + fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], trades) store_plot_file(fig, filename='freqtrade-profit-plot.html', auto_open=True) From 44e050095859004c0c2a913027cc5eee01d71a08 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 13:01:12 +0200 Subject: [PATCH 23/25] Test init_plotscript --- freqtrade/tests/test_plotting.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index 32e7dcd8a..cef229d19 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -11,8 +11,8 @@ from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data from freqtrade.plot.plotting import (add_indicators, add_profit, generate_candlestick_graph, generate_plot_filename, - generate_profit_graph, plot_trades, - store_plot_file) + generate_profit_graph, init_plotscript, + plot_trades, store_plot_file) from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.tests.conftest import log_has, log_has_re @@ -37,6 +37,26 @@ def generage_empty_figure(): ) +def test_init_plotscript(default_conf, mocker): + default_conf['timerange'] = "20180110-20180112" + default_conf['trade_source'] = "file" + default_conf['ticker_interval'] = "5m" + default_conf["datadir"] = history.make_testdata_path(None) + default_conf['exportfilename'] = str( + history.make_testdata_path(None) / "backtest-result_test.json") + ret = init_plotscript(default_conf) + assert "tickers" in ret + assert "trades" in ret + assert "pairs" in ret + assert "strategy" in ret + + default_conf['pairs'] = "POWR/BTC,XLM/BTC" + ret = init_plotscript(default_conf) + assert "tickers" in ret + assert "POWR/BTC" in ret["tickers"] + assert "XLM/BTC" in ret["tickers"] + + def test_add_indicators(default_conf, caplog): pair = "UNITTEST/BTC" timerange = TimeRange(None, 'line', 0, -1000) From 59818af69ced404503ab543c4ce03752c6016082 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Jun 2019 13:15:41 +0200 Subject: [PATCH 24/25] Remove common_datearray function --- freqtrade/misc.py | 20 -------------------- freqtrade/plot/plotting.py | 4 ++-- freqtrade/tests/test_misc.py | 21 +++------------------ 3 files changed, 5 insertions(+), 40 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 460e20e91..05946e008 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -5,10 +5,8 @@ import gzip import logging import re from datetime import datetime -from typing import Dict import numpy as np -from pandas import DataFrame import rapidjson @@ -41,24 +39,6 @@ def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray: return dates.dt.to_pydatetime() -def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray: - """ - Return dates from Dataframe - :param dfs: Dict with format pair: pair_data - :return: List of dates - """ - alldates = {} - for pair, pair_data in dfs.items(): - dates = datesarray_to_datetimearray(pair_data['date']) - for date in dates: - alldates[date] = 1 - lst = [] - for date, _ in alldates.items(): - lst.append(date) - arr = np.array(lst) - return np.sort(arr, axis=0) - - def file_dump_json(filename, data, is_zip=False) -> None: """ Dump JSON data into a file diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 079a098dc..ccb932698 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -266,8 +266,8 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra return fig -def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], trades: pd.DataFrame = None, - ) -> go.Figure: +def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], + trades: pd.DataFrame) -> go.Figure: # Combine close-values for all pairs, rename columns to "pair" df_comb = combine_tickers_with_mean(tickers, "close") diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 7a7b15cf2..1a6b2a92d 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -4,10 +4,9 @@ import datetime from unittest.mock import MagicMock from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, - file_dump_json, file_load_json, format_ms_time, shorten_date) -from freqtrade.data.history import load_tickerdata_file, pair_data_filename -from freqtrade.strategy.default_strategy import DefaultStrategy +from freqtrade.data.history import pair_data_filename +from freqtrade.misc import (datesarray_to_datetimearray, file_dump_json, + file_load_json, format_ms_time, shorten_date) def test_shorten_date() -> None: @@ -32,20 +31,6 @@ def test_datesarray_to_datetimearray(ticker_history_list): assert date_len == 2 -def test_common_datearray(default_conf) -> None: - strategy = DefaultStrategy(default_conf) - tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, "1m", pair="UNITTEST/BTC", - fill_missing=True)} - dataframes = strategy.tickerdata_to_dataframe(tickerlist) - - dates = common_datearray(dataframes) - - assert dates.size == dataframes['UNITTEST/BTC']['date'].size - assert dates[0] == dataframes['UNITTEST/BTC']['date'][0] - assert dates[-1] == dataframes['UNITTEST/BTC']['date'].iloc[-1] - - def test_file_dump_json(mocker) -> None: file_open = mocker.patch('freqtrade.misc.open', MagicMock()) json_dump = mocker.patch('rapidjson.dump', MagicMock()) From b3644f7fa000e2be4daabceab47216175552a1ca Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 3 Jul 2019 06:26:39 +0200 Subject: [PATCH 25/25] Fix typo in docstring --- freqtrade/data/btanalysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 556666f4e..dcd544d00 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -106,7 +106,7 @@ def load_trades(config) -> pd.DataFrame: """ Based on configuration option "trade_source": * loads data from DB (using `db_url`) - * loads data from backtestfile (`using exportfilename`) + * loads data from backtestfile (using `exportfilename`) """ if config["trade_source"] == "DB": return load_trades_from_db(config["db_url"])