From 27faf12fdeef84f404aa7bf32f27e7591cc1a910 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Sat, 14 Mar 2020 22:15:03 +0100 Subject: [PATCH 1/8] Fix if no file exists --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 5 +++++ freqtrade/data/btanalysis.py | 8 ++++++-- freqtrade/plot/plotting.py | 19 +++++++++++++++---- tests/data/test_btanalysis.py | 5 ++++- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 1a8cca72b..66fa0b038 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -59,7 +59,7 @@ ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchang ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", - "timerange", "ticker_interval"] + "timerange", "ticker_interval", "skip_trades"] ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "ticker_interval"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 42c697d56..187e0a424 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -413,6 +413,11 @@ AVAILABLE_CLI_OPTIONS = { metavar='INT', default=750, ), + "skip_trades": Arg( + '--skip-trades', + help='Skip using trades file from backtesting and DB.', + action='store_true', + ), "trade_source": Arg( '--trade-source', help='Specify the source for trades (Can be DB or file (backtest file)) ' diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 681bf6734..fc938ad7e 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -111,7 +111,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: t.calc_profit(), t.calc_profit_ratio(), t.open_rate, t.close_rate, t.amount, (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2) - if t.close_date else None), + if t.close_date else None), t.sell_reason, t.fee_open, t.fee_close, t.open_rate_requested, @@ -129,12 +129,16 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: return trades -def load_trades(source: str, db_url: str, exportfilename: str) -> pd.DataFrame: +def load_trades(source: str, db_url: str, exportfilename: str, skip_trades: bool) -> pd.DataFrame: """ Based on configuration option "trade_source": * loads data from DB (using `db_url`) * loads data from backtestfile (using `exportfilename`) """ + if skip_trades: + df = pd.DataFrame(columns=BT_DATA_COLUMNS) + return df + if source == "DB": return load_trades_from_db(db_url) elif source == "file": diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index be7be2de0..cb2686878 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import Any, Dict, List import pandas as pd +from os.path import isfile from freqtrade.configuration import TimeRange from freqtrade.data.btanalysis import (calculate_max_drawdown, @@ -48,11 +49,21 @@ def init_plotscript(config): data_format=config.get('dataformat_ohlcv', 'json'), ) - trades = load_trades(config['trade_source'], - db_url=config.get('db_url'), - exportfilename=config.get('exportfilename'), - ) + skip_trades = False + if not isfile(config.get('exportfilename')) and config['trade_source'] == 'file': + logger.info("Backtest file is missing skipping trades.") + skip_trades = True + elif config.get('skip_trades', False): + skip_trades = True + + trades = load_trades( + config['trade_source'], + db_url=config.get('db_url'), + exportfilename=config.get('exportfilename'), + skip_trades=skip_trades + ) trades = trim_dataframe(trades, timerange, 'open_time') + return {"ohlcv": data, "trades": trades, "pairs": pairs, diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index da5d225b9..44e9c1072 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -104,6 +104,7 @@ def test_load_trades(default_conf, mocker): load_trades("DB", db_url=default_conf.get('db_url'), exportfilename=default_conf.get('exportfilename'), + skip_trades=False ) assert db_mock.call_count == 1 @@ -114,7 +115,9 @@ def test_load_trades(default_conf, mocker): default_conf['exportfilename'] = "testfile.json" load_trades("file", db_url=default_conf.get('db_url'), - exportfilename=default_conf.get('exportfilename'),) + exportfilename=default_conf.get('exportfilename'), + skip_trades=False + ) assert db_mock.call_count == 0 assert bt_mock.call_count == 1 From cf7e80f45d5d664c1dcb1cc92335cffb67186d12 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Sat, 14 Mar 2020 23:55:13 +0100 Subject: [PATCH 2/8] Docs and logging --- docs/plotting.md | 62 +++++++++++++++++++++++++------------- freqtrade/plot/plotting.py | 2 +- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/docs/plotting.md b/docs/plotting.md index 3eef8f8e7..ef28f0fe3 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -23,44 +23,64 @@ The `freqtrade plot-dataframe` subcommand shows an interactive graph with three Possible arguments: ``` -usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] - [--strategy-path PATH] [-p PAIRS [PAIRS ...]] [--indicators1 INDICATORS1 [INDICATORS1 ...]] - [--indicators2 INDICATORS2 [INDICATORS2 ...]] [--plot-limit INT] [--db-url PATH] - [--trade-source {DB,file}] [--export EXPORT] [--export-filename PATH] [--timerange TIMERANGE] - [-i TICKER_INTERVAL] +usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [-s NAME] + [--strategy-path PATH] [-p PAIRS [PAIRS ...]] + [--indicators1 INDICATORS1 [INDICATORS1 ...]] + [--indicators2 INDICATORS2 [INDICATORS2 ...]] + [--plot-limit INT] [--db-url PATH] + [--trade-source {DB,file}] [--export EXPORT] + [--export-filename PATH] + [--timerange TIMERANGE] [-i TICKER_INTERVAL] + [--skip-trades] optional arguments: -h, --help show this help message and exit -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] - Show profits for only these pairs. Pairs are space-separated. + Show profits for only these pairs. Pairs are space- + separated. --indicators1 INDICATORS1 [INDICATORS1 ...] - Set indicators from your strategy you want in the first row of the graph. Space-separated list. Example: + Set indicators from your strategy you want in the + first row of the graph. Space-separated list. Example: `ema3 ema5`. Default: `['sma', 'ema3', 'ema5']`. --indicators2 INDICATORS2 [INDICATORS2 ...] - Set indicators from your strategy you want in the third row of the graph. Space-separated list. Example: + Set indicators from your strategy you want in the + third row of the graph. Space-separated list. Example: `fastd fastk`. Default: `['macd', 'macdsignal']`. - --plot-limit INT Specify tick limit for plotting. Notice: too high values cause huge files. Default: 750. - --db-url PATH Override trades database URL, this is useful in custom deployments (default: `sqlite:///tradesv3.sqlite` - for Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for Dry Run). + --plot-limit INT Specify tick limit for plotting. Notice: too high + values cause huge files. Default: 750. + --db-url PATH Override trades database URL, this is useful in custom + deployments (default: `sqlite:///tradesv3.sqlite` for + Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for + Dry Run). --trade-source {DB,file} - Specify the source for trades (Can be DB or file (backtest file)) Default: file - --export EXPORT Export backtest results, argument are: trades. Example: `--export=trades` + Specify the source for trades (Can be DB or file + (backtest file)) Default: file + --export EXPORT Export backtest results, argument are: trades. + Example: `--export=trades` --export-filename PATH - Save backtest results to the file with this filename. Requires `--export` to be set as well. Example: - `--export-filename=user_data/backtest_results/backtest_today.json` + Save backtest results to the file with this filename. + Requires `--export` to be set as well. Example: + `--export-filename=user_data/backtest_results/backtest + _today.json` --timerange TIMERANGE Specify what timerange of data to use. -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL - Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`). + Specify ticker interval (`1m`, `5m`, `30m`, `1h`, + `1d`). + --skip-trades Skip using trades file from backtesting and DB. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH @@ -68,9 +88,9 @@ Common arguments: Strategy arguments: -s NAME, --strategy NAME - Specify strategy class name which will be used by the bot. + Specify strategy class name which will be used by the + bot. --strategy-path PATH Specify additional strategy lookup path. - ``` Example: diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index cb2686878..8da61597f 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -51,7 +51,7 @@ def init_plotscript(config): skip_trades = False if not isfile(config.get('exportfilename')) and config['trade_source'] == 'file': - logger.info("Backtest file is missing skipping trades.") + logger.warning("Backtest file is missing skipping trades.") skip_trades = True elif config.get('skip_trades', False): skip_trades = True From 2c0980aa3ab2af40e46411819f2e53b76eae6fa4 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Sun, 15 Mar 2020 00:09:08 +0100 Subject: [PATCH 3/8] Tests --- tests/data/test_btanalysis.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 44e9c1072..95f371d7f 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -122,6 +122,18 @@ def test_load_trades(default_conf, mocker): assert db_mock.call_count == 0 assert bt_mock.call_count == 1 + db_mock.reset_mock() + bt_mock.reset_mock() + default_conf['exportfilename'] = "testfile.json" + load_trades("file", + db_url=default_conf.get('db_url'), + exportfilename=default_conf.get('exportfilename'), + skip_trades=True + ) + + assert db_mock.call_count == 0 + assert bt_mock.call_count == 0 + def test_combine_dataframes_with_mean(testdatadir): pairs = ["ETH/BTC", "ADA/BTC"] From 8c33e07dc660853eee0eee0d10109063f52d11cf Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Sun, 15 Mar 2020 21:20:32 +0100 Subject: [PATCH 4/8] Update based on comments --- docs/plotting.md | 2 +- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 6 +++--- freqtrade/data/btanalysis.py | 6 +++--- freqtrade/plot/plotting.py | 13 ++++++------- tests/data/test_btanalysis.py | 6 +++--- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/docs/plotting.md b/docs/plotting.md index ef28f0fe3..56906ebec 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -68,7 +68,7 @@ optional arguments: -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`). - --skip-trades Skip using trades file from backtesting and DB. + --no-trades Skip using trades from backtesting file and DB. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 66fa0b038..8c64c5857 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -59,7 +59,7 @@ ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchang ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", - "timerange", "ticker_interval", "skip_trades"] + "timerange", "ticker_interval", "no_trades"] ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "ticker_interval"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 187e0a424..5cf1b7fce 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -413,9 +413,9 @@ AVAILABLE_CLI_OPTIONS = { metavar='INT', default=750, ), - "skip_trades": Arg( - '--skip-trades', - help='Skip using trades file from backtesting and DB.', + "no_trades": Arg( + '--no-trades', + help='Skip using trades from backtesting file and DB.', action='store_true', ), "trade_source": Arg( diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 8670f30c6..17b243f56 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -129,7 +129,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: return trades -def load_trades(source: str, db_url: str, exportfilename: Path, skip_trades: bool) -> pd.DataFrame: +def load_trades(source: str, db_url: str, exportfilename: Path, no_trades: bool) -> pd.DataFrame: """ Based on configuration option "trade_source": * loads data from DB (using `db_url`) @@ -137,10 +137,10 @@ def load_trades(source: str, db_url: str, exportfilename: Path, skip_trades: boo :param source: "DB" or "file" - specify source to load from :param db_url: sqlalchemy formatted url to a database :param exportfilename: Json file generated by backtesting - :param skip_trades: Skip using trades, only return backtesting data columns + :param no_trades: Skip using trades, only return backtesting data columns :return: DataFrame containing trades """ - if skip_trades: + if no_trades: df = pd.DataFrame(columns=BT_DATA_COLUMNS) return df diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 8da61597f..fc8f25612 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -3,7 +3,6 @@ from pathlib import Path from typing import Any, Dict, List import pandas as pd -from os.path import isfile from freqtrade.configuration import TimeRange from freqtrade.data.btanalysis import (calculate_max_drawdown, @@ -49,18 +48,18 @@ def init_plotscript(config): data_format=config.get('dataformat_ohlcv', 'json'), ) - skip_trades = False - if not isfile(config.get('exportfilename')) and config['trade_source'] == 'file': + no_trades = False + if config.get('no_trades', False): + no_trades = True + elif not config['exportfilename'].is_file() and config['trade_source'] == 'file': logger.warning("Backtest file is missing skipping trades.") - skip_trades = True - elif config.get('skip_trades', False): - skip_trades = True + no_trades = True trades = load_trades( config['trade_source'], db_url=config.get('db_url'), exportfilename=config.get('exportfilename'), - skip_trades=skip_trades + no_trades=no_trades ) trades = trim_dataframe(trades, timerange, 'open_time') diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 8561f2b84..4b47faeee 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -105,7 +105,7 @@ def test_load_trades(default_conf, mocker): load_trades("DB", db_url=default_conf.get('db_url'), exportfilename=default_conf.get('exportfilename'), - skip_trades=False + no_trades=False ) assert db_mock.call_count == 1 @@ -117,7 +117,7 @@ def test_load_trades(default_conf, mocker): load_trades("file", db_url=default_conf.get('db_url'), exportfilename=default_conf.get('exportfilename'), - skip_trades=False + no_trades=False ) assert db_mock.call_count == 0 @@ -129,7 +129,7 @@ def test_load_trades(default_conf, mocker): load_trades("file", db_url=default_conf.get('db_url'), exportfilename=default_conf.get('exportfilename'), - skip_trades=True + no_trades=True ) assert db_mock.call_count == 0 From 06198c00283eeaef44f0dda391a3cc769acce6ae Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Sun, 15 Mar 2020 21:27:45 +0100 Subject: [PATCH 5/8] Missed configuration.py --- freqtrade/configuration/configuration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index ce2101441..e5515670d 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -359,6 +359,9 @@ class Configuration: self._args_to_config(config, argname='erase', logstring='Erase detected. Deleting existing data.') + self._args_to_config(config, argname='no_trades', + logstring='Parameter --no-trades detected.') + self._args_to_config(config, argname='timeframes', logstring='timeframes --timeframes: {}') From 05250ba6611278b794677780a2e00febdb14f476 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Wed, 18 Mar 2020 11:00:33 +0100 Subject: [PATCH 6/8] Update docs/plotting.md Co-Authored-By: Matthias --- docs/plotting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plotting.md b/docs/plotting.md index 56906ebec..be83065a6 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -32,7 +32,7 @@ usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--trade-source {DB,file}] [--export EXPORT] [--export-filename PATH] [--timerange TIMERANGE] [-i TICKER_INTERVAL] - [--skip-trades] + [--no-trades] optional arguments: -h, --help show this help message and exit From 0920d6fce4d35ebd3f99ed321131764fde628206 Mon Sep 17 00:00:00 2001 From: Fredrik81 Date: Wed, 18 Mar 2020 11:01:09 +0100 Subject: [PATCH 7/8] Update freqtrade/data/btanalysis.py Co-Authored-By: Matthias --- freqtrade/data/btanalysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 17b243f56..898072748 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -129,7 +129,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: return trades -def load_trades(source: str, db_url: str, exportfilename: Path, no_trades: bool) -> pd.DataFrame: +def load_trades(source: str, db_url: str, exportfilename: Path, no_trades: bool = False) -> pd.DataFrame: """ Based on configuration option "trade_source": * loads data from DB (using `db_url`) From 3e1bef888ab1344f8763e130cdef835b0b030d6b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Mar 2020 11:42:42 +0100 Subject: [PATCH 8/8] Fix flake8 error --- freqtrade/data/btanalysis.py | 3 ++- tests/data/test_btanalysis.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 898072748..23a9f720c 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -129,7 +129,8 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: return trades -def load_trades(source: str, db_url: str, exportfilename: Path, no_trades: bool = False) -> pd.DataFrame: +def load_trades(source: str, db_url: str, exportfilename: Path, + no_trades: bool = False) -> pd.DataFrame: """ Based on configuration option "trade_source": * loads data from DB (using `db_url`) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 4b47faeee..463e5ae36 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -117,7 +117,6 @@ def test_load_trades(default_conf, mocker): load_trades("file", db_url=default_conf.get('db_url'), exportfilename=default_conf.get('exportfilename'), - no_trades=False ) assert db_mock.call_count == 0