adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2176 from freqtrade/plot_commands

Browse Source 
Move Plot scripts to freqtrade subcommands
This commit is contained in:
hroff-1902
2019-09-02 08:08:51 +03:00
committed by GitHub
parent c64beb3f76 2886fa288a
commit 08b090c707
18 changed files with 415 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
freqtrade/configuration/arguments.py
View File

@@ -34,15 +34,13 @@ ARGS_CREATE_USERDIR = ["user_data_dir"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"]
ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY +
["pairs", "indicators1", "indicators2", "plot_limit", "db_url",
"trade_source", "export", "exportfilename", "timerange",
"refresh_pairs"])
ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url",
"trade_source", "export", "exportfilename", "timerange", "ticker_interval"]
ARGS_PLOT_PROFIT = (ARGS_COMMON + ARGS_STRATEGY +
["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source"])
ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
"trade_source", "ticker_interval"]
NO_CONF_REQURIED = ["start_download_data"]
NO_CONF_REQURIED = ["download-data", "plot-dataframe", "plot-profit"]
class Arguments(object):
@@ -81,8 +79,7 @@ class Arguments(object):
# (see https://bugs.python.org/issue16399)
# Allow no-config for certain commands (like downloading / plotting)
if (not self._no_default_config and parsed_arg.config is None
and not (hasattr(parsed_arg, 'func')
and parsed_arg.func.__name__ in NO_CONF_REQURIED)):
and not ('subparser' in parsed_arg and parsed_arg.subparser in NO_CONF_REQURIED)):
parsed_arg.config = [constants.DEFAULT_CONFIG]
return parsed_arg
@@ -119,6 +116,7 @@ class Arguments(object):
hyperopt_cmd.set_defaults(func=start_hyperopt)
self._build_args(optionlist=ARGS_HYPEROPT, parser=hyperopt_cmd)
# add create-userdir subcommand
create_userdir_cmd = subparsers.add_parser('create-userdir',
help="Create user-data directory.")
create_userdir_cmd.set_defaults(func=start_create_userdir)
@@ -139,3 +137,20 @@ class Arguments(object):
)
download_data_cmd.set_defaults(func=start_download_data)
self._build_args(optionlist=ARGS_DOWNLOAD_DATA, parser=download_data_cmd)
# Add Plotting subcommand
from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit
plot_dataframe_cmd = subparsers.add_parser(
'plot-dataframe',
help='Plot candles with indicators.'
)
plot_dataframe_cmd.set_defaults(func=start_plot_dataframe)
self._build_args(optionlist=ARGS_PLOT_DATAFRAME, parser=plot_dataframe_cmd)
# Plot profit
plot_profit_cmd = subparsers.add_parser(
'plot-profit',
help='Generate plot showing profits.'
)
plot_profit_cmd.set_defaults(func=start_plot_profit)
self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)

5
freqtrade/configuration/check_exchange.py
View File

@@ -5,6 +5,7 @@ from freqtrade import OperationalException
from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason,
is_exchange_available, is_exchange_bad,
is_exchange_officially_supported)
from freqtrade.state import RunMode
logger = logging.getLogger(__name__)
@@ -19,6 +20,10 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
raises an exception if the exchange if not supported by ccxt
and thus is not known for the Freqtrade at all.
"""
if config['runmode'] in [RunMode.PLOT] and not config.get('exchange', {}).get('name'):
# Skip checking exchange in plot mode, since it requires no exchange
return True
logger.info("Checking exchange...")
exchange = config.get('exchange', {}).get('name').lower()

10
freqtrade/configuration/cli_options.py
View File

@@ -292,14 +292,16 @@ AVAILABLE_CLI_OPTIONS = {
"indicators1": Arg(
'--indicators1',
help='Set indicators from your strategy you want in the first row of the graph. '
'Comma-separated list. Example: `ema3,ema5`. Default: `%(default)s`.',
default='sma,ema3,ema5',
'Space-separated list. Example: `ema3 ema5`. Default: `%(default)s`.',
default=['sma', 'ema3', 'ema5'],
nargs='+',
),
"indicators2": Arg(
'--indicators2',
help='Set indicators from your strategy you want in the third row of the graph. '
'Comma-separated list. Example: `fastd,fastk`. Default: `%(default)s`.',
default='macd,macdsignal',
'Space-separated list. Example: `fastd fastk`. Default: `%(default)s`.',
default=['macd', 'macdsignal'],
nargs='+',
),
"plot_limit": Arg(
'--plot-limit',

13
freqtrade/data/btanalysis.py
View File

@@ -112,16 +112,16 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame:
return trades
def load_trades(config) -> pd.DataFrame:
def load_trades(source: str, db_url: str, exportfilename: str) -> pd.DataFrame:
"""
Based on configuration option "trade_source":
* loads data from DB (using `db_url`)
* loads data from backtestfile (using `exportfilename`)
"""
if config["trade_source"] == "DB":
return load_trades_from_db(config["db_url"])
elif config["trade_source"] == "file":
return load_backtest_data(Path(config["exportfilename"]))
if source == "DB":
return load_trades_from_db(db_url)
elif source == "file":
return load_backtest_data(Path(exportfilename))
def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame:
@@ -157,7 +157,8 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str) ->
:param trades: DataFrame containing trades (requires columns close_time and profitperc)
:return: Returns df with one additional column, col_name, containing the cumulative profit.
"""
df[col_name] = trades.set_index('close_time')['profitperc'].cumsum()
# Use groupby/sum().cumsum() to avoid errors when multiple trades sold at the same candle.
df[col_name] = trades.groupby('close_time')['profitperc'].sum().cumsum()
# Set first value to 0
df.loc[df.iloc[0].name, col_name] = 0
# FFill to get continuous

26
freqtrade/plot/plot_utils.py Normal file
View File

@@ -0,0 +1,26 @@
from argparse import Namespace
from freqtrade.state import RunMode
from freqtrade.utils import setup_utils_configuration
def start_plot_dataframe(args: Namespace) -> None:
"""
Entrypoint for dataframe plotting
"""
# Import here to avoid errors if plot-dependencies are not installed.
from freqtrade.plot.plotting import analyse_and_plot_pairs
config = setup_utils_configuration(args, RunMode.PLOT)
analyse_and_plot_pairs(config)
def start_plot_profit(args: Namespace) -> None:
"""
Entrypoint for plot_profit
"""
# Import here to avoid errors if plot-dependencies are not installed.
from freqtrade.plot.plotting import plot_profit
config = setup_utils_configuration(args, RunMode.PLOT)
plot_profit(config)

105
freqtrade/plot/plotting.py
View File

@@ -1,15 +1,15 @@
import logging
from pathlib import Path
from typing import Dict, List, Optional
from typing import Any, Dict, List
import pandas as pd
from freqtrade.configuration import TimeRange
from freqtrade.data import history
from freqtrade.data.btanalysis import (combine_tickers_with_mean,
create_cum_profit, load_trades)
from freqtrade.exchange import Exchange
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
create_cum_profit,
extract_trades_of_period, load_trades)
from freqtrade.resolvers import StrategyResolver
logger = logging.getLogger(__name__)
@@ -19,23 +19,16 @@ try:
from plotly.offline import plot
import plotly.graph_objects as go
except ImportError:
logger.exception("Module plotly not found \n Please install using `pip install plotly`")
logger.exception("Module plotly not found \n Please install using `pip3 install plotly`")
exit(1)
def init_plotscript(config):
"""
Initialize objects needed for plotting
:return: Dict with tickers, trades, pairs and strategy
:return: Dict with tickers, trades and pairs
"""
exchange: Optional[Exchange] = None
# Exchange is only needed when downloading data!
if config.get("refresh_pairs", False):
exchange = ExchangeResolver(config.get('exchange', {}).get('name'),
config).exchange
strategy = StrategyResolver(config).strategy
if "pairs" in config:
pairs = config["pairs"]
else:
@@ -47,17 +40,18 @@ def init_plotscript(config):
tickers = history.load_data(
datadir=Path(str(config.get("datadir"))),
pairs=pairs,
ticker_interval=config['ticker_interval'],
refresh_pairs=config.get('refresh_pairs', False),
ticker_interval=config.get('ticker_interval', '5m'),
timerange=timerange,
exchange=exchange,
)
trades = load_trades(config)
trades = load_trades(config['trade_source'],
db_url=config.get('db_url'),
exportfilename=config.get('exportfilename'),
)
return {"tickers": tickers,
"trades": trades,
"pairs": pairs,
"strategy": strategy,
}
@@ -280,8 +274,15 @@ def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame],
name='Avg close price',
)
fig = make_subplots(rows=3, cols=1, shared_xaxes=True, row_width=[1, 1, 1])
fig['layout'].update(title="Profit plot")
fig = make_subplots(rows=3, cols=1, shared_xaxes=True,
row_width=[1, 1, 1],
vertical_spacing=0.05,
subplot_titles=["AVG Close Price", "Combined Profit", "Profit per pair"])
fig['layout'].update(title="Freqtrade Profit plot")
fig['layout']['yaxis1'].update(title='Price')
fig['layout']['yaxis2'].update(title='Profit')
fig['layout']['yaxis3'].update(title='Profit')
fig['layout']['xaxis']['rangeslider'].update(visible=False)
fig.add_trace(avgclose, 1, 1)
fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit')
@@ -321,3 +322,67 @@ def store_plot_file(fig, filename: str, directory: Path, auto_open: bool = False
plot(fig, filename=str(_filename),
auto_open=auto_open)
logger.info(f"Stored plot as {_filename}")
def analyse_and_plot_pairs(config: Dict[str, Any]):
"""
From configuration provided
- Initializes plot-script
- Get tickers data
- Generate Dafaframes populated with indicators and signals based on configured strategy
- Load trades excecuted during the selected period
- Generate Plotly plot objects
- Generate plot files
:return: None
"""
strategy = StrategyResolver(config).strategy
plot_elements = init_plotscript(config)
trades = plot_elements['trades']
pair_counter = 0
for pair, data in plot_elements["tickers"].items():
pair_counter += 1
logger.info("analyse pair %s", pair)
tickers = {}
tickers[pair] = data
dataframe = strategy.analyze_ticker(tickers[pair], {'pair': pair})
trades_pair = trades.loc[trades['pair'] == pair]
trades_pair = extract_trades_of_period(dataframe, trades_pair)
fig = generate_candlestick_graph(
pair=pair,
data=dataframe,
trades=trades_pair,
indicators1=config["indicators1"],
indicators2=config["indicators2"],
)
store_plot_file(fig, filename=generate_plot_filename(pair, config['ticker_interval']),
directory=config['user_data_dir'] / "plot")
logger.info('End of plotting process. %s plots generated', pair_counter)
def plot_profit(config: Dict[str, Any]) -> None:
"""
Plots the total profit for all pairs.
Note, the profit calculation isn't realistic.
But should be somewhat proportional, and therefor useful
in helping out to find a good algorithm.
"""
plot_elements = init_plotscript(config)
trades = load_trades(config['trade_source'],
db_url=str(config.get('db_url')),
exportfilename=str(config.get('exportfilename')),
)
# Filter trades to relevant pairs
trades = trades[trades['pair'].isin(plot_elements["pairs"])]
# Create an average close price of all the pairs that were involved.
# this could be useful to gauge the overall market trend
fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], trades)
store_plot_file(fig, filename='freqtrade-profit-plot.html',
directory=config['user_data_dir'] / "plot", auto_open=True)

1
freqtrade/state.py
View File

@@ -25,4 +25,5 @@ class RunMode(Enum):
BACKTEST = "backtest"
EDGE = "edge"
HYPEROPT = "hyperopt"
PLOT = "plot"
OTHER = "other" # Used for plotting scripts and test

11
freqtrade/tests/data/test_btanalysis.py
View File

@@ -89,17 +89,20 @@ def test_load_trades(default_conf, mocker):
db_mock = mocker.patch("freqtrade.data.btanalysis.load_trades_from_db", MagicMock())
bt_mock = mocker.patch("freqtrade.data.btanalysis.load_backtest_data", MagicMock())
default_conf['trade_source'] = "DB"
load_trades(default_conf)
load_trades("DB",
db_url=default_conf.get('db_url'),
exportfilename=default_conf.get('exportfilename'),
)
assert db_mock.call_count == 1
assert bt_mock.call_count == 0
db_mock.reset_mock()
bt_mock.reset_mock()
default_conf['trade_source'] = "file"
default_conf['exportfilename'] = "testfile.json"
load_trades(default_conf)
load_trades("file",
db_url=default_conf.get('db_url'),
exportfilename=default_conf.get('exportfilename'),)
assert db_mock.call_count == 0
assert bt_mock.call_count == 1

30
freqtrade/tests/test_arguments.py
View File

@@ -4,7 +4,6 @@ import argparse
import pytest
from freqtrade.configuration import Arguments
from freqtrade.configuration.arguments import ARGS_PLOT_DATAFRAME
from freqtrade.configuration.cli_options import check_int_positive
@@ -149,20 +148,35 @@ def test_download_data_options() -> None:
def test_plot_dataframe_options() -> None:
args = [
'--indicators1', 'sma10,sma100',
'--indicators2', 'macd,fastd,fastk',
'-c', 'config.json.example',
'plot-dataframe',
'--indicators1', 'sma10', 'sma100',
'--indicators2', 'macd', 'fastd', 'fastk',
'--plot-limit', '30',
'-p', 'UNITTEST/BTC',
]
arguments = Arguments(args, '')
arguments._build_args(ARGS_PLOT_DATAFRAME)
pargs = arguments._parse_args()
assert pargs.indicators1 == "sma10,sma100"
assert pargs.indicators2 == "macd,fastd,fastk"
pargs = Arguments(args, '').get_parsed_arg()
assert pargs.indicators1 == ["sma10", "sma100"]
assert pargs.indicators2 == ["macd", "fastd", "fastk"]
assert pargs.plot_limit == 30
assert pargs.pairs == ["UNITTEST/BTC"]
def test_plot_profit_options() -> None:
args = [
'plot-profit',
'-p', 'UNITTEST/BTC',
'--trade-source', 'DB',
"--db-url", "sqlite:///whatever.sqlite",
]
pargs = Arguments(args, '').get_parsed_arg()
assert pargs.trade_source == "DB"
assert pargs.pairs == ["UNITTEST/BTC"]
assert pargs.db_url == "sqlite:///whatever.sqlite"
def test_check_int_positive() -> None:
assert check_int_positive("3") == 3
assert check_int_positive("1") == 1

6
freqtrade/tests/test_configuration.py
View File

@@ -479,6 +479,7 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None:
def test_check_exchange(default_conf, caplog) -> None:
# Test an officially supported by Freqtrade team exchange
default_conf['runmode'] = RunMode.DRY_RUN
default_conf.get('exchange').update({'name': 'BITTREX'})
assert check_exchange(default_conf)
assert log_has_re(r"Exchange .* is officially supported by the Freqtrade development team\.",
@@ -523,6 +524,11 @@ def test_check_exchange(default_conf, caplog) -> None:
):
check_exchange(default_conf)
# Test no exchange...
default_conf.get('exchange').update({'name': ''})
default_conf['runmode'] = RunMode.PLOT
assert check_exchange(default_conf)
def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None:
patched_configuration_load_config_file(mocker, default_conf)

95
freqtrade/tests/test_plotting.py
View File

@@ -9,13 +9,15 @@ from plotly.subplots import make_subplots
from freqtrade.configuration import TimeRange
from freqtrade.data import history
from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data
from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit
from freqtrade.plot.plotting import (add_indicators, add_profit,
analyse_and_plot_pairs,
generate_candlestick_graph,
generate_plot_filename,
generate_profit_graph, init_plotscript,
plot_trades, store_plot_file)
plot_profit, plot_trades, store_plot_file)
from freqtrade.strategy.default_strategy import DefaultStrategy
from freqtrade.tests.conftest import log_has, log_has_re
from freqtrade.tests.conftest import get_args, log_has, log_has_re
def fig_generating_mock(fig, *args, **kwargs):
@@ -49,7 +51,6 @@ def test_init_plotscript(default_conf, mocker):
assert "tickers" in ret
assert "trades" in ret
assert "pairs" in ret
assert "strategy" in ret
default_conf['pairs'] = ["POWR/BTC", "XLM/BTC"]
ret = init_plotscript(default_conf)
@@ -257,7 +258,11 @@ def test_generate_profit_graph():
fig = generate_profit_graph(pairs, tickers, trades)
assert isinstance(fig, go.Figure)
assert fig.layout.title.text == "Profit plot"
assert fig.layout.title.text == "Freqtrade Profit plot"
assert fig.layout.yaxis.title.text == "Price"
assert fig.layout.yaxis2.title.text == "Profit"
assert fig.layout.yaxis3.title.text == "Profit"
figure = fig.layout.figure
assert len(figure.data) == 4
@@ -270,3 +275,85 @@ def test_generate_profit_graph():
for pair in pairs:
profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}")
assert isinstance(profit_pair, go.Scattergl)
def test_start_plot_dataframe(mocker):
aup = mocker.patch("freqtrade.plot.plotting.analyse_and_plot_pairs", MagicMock())
args = [
"--config", "config.json.example",
"plot-dataframe",
"--pairs", "ETH/BTC"
]
start_plot_dataframe(get_args(args))
assert aup.call_count == 1
called_config = aup.call_args_list[0][0][0]
assert "pairs" in called_config
assert called_config['pairs'] == ["ETH/BTC"]
def test_analyse_and_plot_pairs(default_conf, mocker, caplog):
default_conf['trade_source'] = 'file'
default_conf["datadir"] = history.make_testdata_path(None)
default_conf['exportfilename'] = str(
history.make_testdata_path(None) / "backtest-result_test.json")
default_conf['indicators1'] = ["sma5", "ema10"]
default_conf['indicators2'] = ["macd"]
default_conf['pairs'] = ["ETH/BTC", "LTC/BTC"]
candle_mock = MagicMock()
store_mock = MagicMock()
mocker.patch.multiple(
"freqtrade.plot.plotting",
generate_candlestick_graph=candle_mock,
store_plot_file=store_mock
)
analyse_and_plot_pairs(default_conf)
# Both mocks should be called once per pair
assert candle_mock.call_count == 2
assert store_mock.call_count == 2
assert candle_mock.call_args_list[0][1]['indicators1'] == ['sma5', 'ema10']
assert candle_mock.call_args_list[0][1]['indicators2'] == ['macd']
assert log_has("End of plotting process. 2 plots generated", caplog)
def test_start_plot_profit(mocker):
aup = mocker.patch("freqtrade.plot.plotting.plot_profit", MagicMock())
args = [
"--config", "config.json.example",
"plot-profit",
"--pairs", "ETH/BTC"
]
start_plot_profit(get_args(args))
assert aup.call_count == 1
called_config = aup.call_args_list[0][0][0]
assert "pairs" in called_config
assert called_config['pairs'] == ["ETH/BTC"]
def test_plot_profit(default_conf, mocker, caplog):
default_conf['trade_source'] = 'file'
default_conf["datadir"] = history.make_testdata_path(None)
default_conf['exportfilename'] = str(
history.make_testdata_path(None) / "backtest-result_test.json")
default_conf['pairs'] = ["ETH/BTC", "LTC/BTC"]
profit_mock = MagicMock()
store_mock = MagicMock()
mocker.patch.multiple(
"freqtrade.plot.plotting",
generate_profit_graph=profit_mock,
store_plot_file=store_mock
)
plot_profit(default_conf)
# Plot-profit generates one combined plot
assert profit_mock.call_count == 1
assert store_mock.call_count == 1
assert profit_mock.call_args_list[0][0][0] == default_conf['pairs']
assert store_mock.call_args_list[0][1]['auto_open'] is True

3
freqtrade/utils.py
View File

@@ -92,6 +92,3 @@ def start_download_data(args: Namespace) -> None:
if pairs_not_available:
logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
f"on exchange {config['exchange']['name']}.")
# configuration.resolve_pairs_list()
print(config)