From 349c0619aa30dba6fe38d70575cea7b710b23e27 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 May 2019 20:06:26 +0200 Subject: [PATCH 01/70] Move startup to freqtradebot --- freqtrade/freqtradebot.py | 8 ++++++++ freqtrade/worker.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8b29d6d40..425b7e3a9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -89,6 +89,14 @@ class FreqtradeBot(object): self.rpc.cleanup() persistence.cleanup() + def startup(self) -> None: + """ + Called on startup and after reloading the bot - triggers notifications and + performs startup tasks + : return: None + """ + self.rpc.startup_messages(self.config, self.pairlists) + def process(self) -> bool: """ Queries the persistence layer for open trades and handles them, diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 19a570505..c224b4ee5 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -91,7 +91,7 @@ class Worker(object): }) logger.info('Changing state to: %s', state.name) if state == State.RUNNING: - self.freqtrade.rpc.startup_messages(self._config, self.freqtrade.pairlists) + self.freqtrade.startup() if state == State.STOPPED: # Ping systemd watchdog before sleeping in the stopped state From 6a5daab520b4f26181abafe44a024777dfafa181 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 May 2019 07:06:40 +0200 Subject: [PATCH 02/70] add logic for stoploss reinitialization after startup --- freqtrade/freqtradebot.py | 2 ++ freqtrade/persistence.py | 20 ++++++++++++++++++++ freqtrade/tests/test_freqtradebot.py | 1 + 3 files changed, 23 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 425b7e3a9..7d5cddd6c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -96,6 +96,8 @@ class FreqtradeBot(object): : return: None """ self.rpc.startup_messages(self.config, self.pairlists) + # Adjust stoploss if it was changed + Trade.stoploss_reinitialization(self.strategy.stoploss) def process(self) -> bool: """ diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index e64e0b89c..3fc9a189e 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -421,3 +421,23 @@ class Trade(_DECL_BASE): Query trades from persistence layer """ return Trade.query.filter(Trade.is_open.is_(True)).all() + + @staticmethod + def stoploss_reinitialization(desired_stoploss): + """ + Adjust initial Stoploss to desired stoploss for all open trades. + """ + for trade in Trade.get_open_trades(): + logger.info("Found open trade: %s", trade) + + # skip case if trailing-stop changed the stoploss already. + if (trade.stop_loss == trade.initial_stop_loss + and trade.initial_stop_loss_pct != desired_stoploss): + # Stoploss value got changed + + logger.info(f"Stoploss for {trade} needs adjustment.") + logger.info(f"Stoploss: {trade.initial_stop_loss_pct}: {desired_stoploss}") + # Force reset of stoploss + trade.stop_loss = None + trade.adjust_stop_loss(trade.open_rate, desired_stoploss) + logger.info(f"new stoploss: {trade.stop_loss}, ") diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 67b05ac3e..c683f6273 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -114,6 +114,7 @@ def test_cleanup(mocker, default_conf, caplog) -> None: def test_worker_running(mocker, default_conf, caplog) -> None: mock_throttle = MagicMock() mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) + mocker.patch('freqtrade.persistence.Trade.adjust_initial_stoploss', MagicMock()) worker = get_patched_worker(mocker, default_conf) From 9f54181494a7b211a7f0cd4213150c36f4193634 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 May 2019 07:12:54 +0200 Subject: [PATCH 03/70] Add test for stoploss_reinit --- freqtrade/tests/test_persistence.py | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 8c15fa8e8..93bad0797 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -777,3 +777,64 @@ def test_to_json(default_conf, fee): 'stop_loss_pct': None, 'initial_stop_loss': None, 'initial_stop_loss_pct': None} + + +def test_stoploss_reinitialization(default_conf, fee): + init(default_conf) + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + fee_open=fee.return_value, + open_date=arrow.utcnow().shift(hours=-2).datetime, + amount=10, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + max_rate=1, + ) + + trade.adjust_stop_loss(trade.open_rate, 0.05, True) + assert trade.stop_loss == 0.95 + assert trade.stop_loss_pct == -0.05 + assert trade.initial_stop_loss == 0.95 + assert trade.initial_stop_loss_pct == -0.05 + Trade.session.add(trade) + + # Lower stoploss + Trade.stoploss_reinitialization(0.06) + + trades = Trade.get_open_trades() + assert len(trades) == 1 + trade_adj = trades[0] + assert trade_adj.stop_loss == 0.94 + assert trade_adj.stop_loss_pct == -0.06 + assert trade_adj.initial_stop_loss == 0.94 + assert trade_adj.initial_stop_loss_pct == -0.06 + + # Raise stoploss + Trade.stoploss_reinitialization(0.04) + + trades = Trade.get_open_trades() + assert len(trades) == 1 + trade_adj = trades[0] + assert trade_adj.stop_loss == 0.96 + assert trade_adj.stop_loss_pct == -0.04 + assert trade_adj.initial_stop_loss == 0.96 + assert trade_adj.initial_stop_loss_pct == -0.04 + + + # Trailing stoploss (move stoplos up a bit) + trade.adjust_stop_loss(1.02, 0.04) + assert trade_adj.stop_loss == 0.9792 + assert trade_adj.initial_stop_loss == 0.96 + + Trade.stoploss_reinitialization(0.04) + + trades = Trade.get_open_trades() + assert len(trades) == 1 + trade_adj = trades[0] + # Stoploss should not change in this case. + assert trade_adj.stop_loss == 0.9792 + assert trade_adj.stop_loss_pct == -0.04 + assert trade_adj.initial_stop_loss == 0.96 + assert trade_adj.initial_stop_loss_pct == -0.04 From 53af8f331d985fc559d64e0d5808f9f57e515d70 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 May 2019 19:35:19 +0200 Subject: [PATCH 04/70] Deep-copy default_conf for edge config --- freqtrade/tests/conftest.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 0bff1d5e9..692fda368 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -2,6 +2,7 @@ import json import logging import re +from copy import deepcopy from datetime import datetime from functools import reduce from unittest.mock import MagicMock, PropertyMock @@ -942,9 +943,10 @@ def buy_order_fee(): @pytest.fixture(scope="function") def edge_conf(default_conf): - default_conf['max_open_trades'] = -1 - default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT - default_conf['edge'] = { + conf = deepcopy(default_conf) + conf['max_open_trades'] = -1 + conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT + conf['edge'] = { "enabled": True, "process_throttle_secs": 1800, "calculate_since_number_of_days": 14, @@ -960,4 +962,4 @@ def edge_conf(default_conf): "remove_pumps": False } - return default_conf + return conf From a39cdd3b2b3b59719bb9b5e53f89f1085776e7dd Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 May 2019 19:35:48 +0200 Subject: [PATCH 05/70] Exclude Edge from startup-stoploss calc Edge would recalculate / reevaluate stoploss values on startup, so these values are not reliable --- freqtrade/freqtradebot.py | 5 +++-- freqtrade/tests/test_freqtradebot.py | 19 ++++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7d5cddd6c..0121512ee 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -96,8 +96,9 @@ class FreqtradeBot(object): : return: None """ self.rpc.startup_messages(self.config, self.pairlists) - # Adjust stoploss if it was changed - Trade.stoploss_reinitialization(self.strategy.stoploss) + if not self.edge: + # Adjust stoploss if it was changed + Trade.stoploss_reinitialization(self.strategy.stoploss) def process(self) -> bool: """ diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index c683f6273..48eb51b54 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -3136,10 +3136,27 @@ def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None: assert rate == 0.043936 -def test_startup_messages(default_conf, mocker): +def test_startup_state(default_conf, mocker): default_conf['pairlist'] = {'method': 'VolumePairList', 'config': {'number_assets': 20} } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) worker = get_patched_worker(mocker, default_conf) assert worker.state is State.RUNNING + + +def test_startup_trade_reinit(default_conf, edge_conf, mocker): + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + reinit_mock = MagicMock() + mocker.patch('freqtrade.persistence.Trade.stoploss_reinitialization', reinit_mock) + + ftbot = get_patched_freqtradebot(mocker, default_conf) + ftbot.startup() + assert reinit_mock.call_count == 1 + + reinit_mock.reset_mock() + + ftbot = get_patched_freqtradebot(mocker, edge_conf) + ftbot.startup() + assert reinit_mock.call_count == 0 From 11fd8a59af3fb92f405e59f3db47f4d82be809f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 May 2019 20:06:13 +0200 Subject: [PATCH 06/70] cleanup stoploss documentations --- docs/configuration.md | 10 ++-------- docs/stoploss.md | 15 ++++++++++++--- docs/strategy-customization.md | 5 ++++- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index df116b3c2..d097712c6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -131,17 +131,11 @@ If it is not set in either Strategy or Configuration, a default of 1000% `{"0": ### Understand stoploss -The `stoploss` configuration parameter is loss in percentage that should trigger a sale. -For example, value `-0.10` will cause immediate sell if the -profit dips below -10% for a given trade. This parameter is optional. - -Most of the strategy files already include the optimal `stoploss` -value. This parameter is optional. If you use it in the configuration file, it will take over the -`stoploss` value from the strategy file. +Go to the [stoploss documentation](stoploss.md) for more details. ### Understand trailing stoploss -Go to the [trailing stoploss Documentation](stoploss.md) for details on trailing stoploss. +Go to the [trailing stoploss Documentation](stoploss.md#trailing-stop-loss) for details on trailing stoploss. ### Understand initial_state diff --git a/docs/stoploss.md b/docs/stoploss.md index cbe4fd3c4..4c731940e 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -1,4 +1,14 @@ -# Stop Loss support +# Stop Loss + +The `stoploss` configuration parameter is loss in percentage that should trigger a sale. +For example, value `-0.10` will cause immediate sell if the profit dips below -10% for a given trade. This parameter is optional. + +Most of the strategy files already include the optimal `stoploss` +value. This parameter is optional. If you use it in the configuration file, it will take over the +`stoploss` value from the strategy file. + + +## Stop Loss support At this stage the bot contains the following stoploss support modes: @@ -16,13 +26,12 @@ In case of stoploss on exchange there is another parameter called `stoploss_on_e !!! Note Stoploss on exchange is only supported for Binance as of now. - ## Static Stop Loss This is very simple, basically you define a stop loss of x in your strategy file or alternative in the configuration, which will overwrite the strategy definition. This will basically try to sell your asset, the second the loss exceeds the defined loss. -## Trail Stop Loss +## Trailing Stop Loss The initial value for this stop loss, is defined in your strategy or configuration. Just as you would define your Stop Loss normally. To enable this Feauture all you have to do is to define the configuration element: diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 51540f690..1256f4a32 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -218,9 +218,12 @@ stoploss = -0.10 ``` This would signify a stoploss of -10%. + +For the full documentation on stoploss features, look at the dedicated [stoploss page](stoploss.md). + If your exchange supports it, it's recommended to also set `"stoploss_on_exchange"` in the order dict, so your stoploss is on the exchange and cannot be missed for network-problems (or other problems). -For more information on order_types please look [here](https://github.com/freqtrade/freqtrade/blob/develop/docs/configuration.md#understand-order_types). +For more information on order_types please look to [here](configuration.md#understand-order_types). ### Ticker interval From 58ced364451c28ba2e6c71fbb5f9f4f6085de975 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 May 2019 20:06:35 +0200 Subject: [PATCH 07/70] Add documentation for stoploss updates --- docs/stoploss.md | 10 ++++++++++ docs/strategy-customization.md | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index 4c731940e..975c2aeb5 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -72,3 +72,13 @@ The 0.01 would translate to a 1% stop loss, once you hit 1.1% profit. You should also make sure to have this value (`trailing_stop_positive_offset`) lower than your minimal ROI, otherwise minimal ROI will apply first and sell your trade. If `"trailing_only_offset_is_reached": true` then the trailing stoploss is only activated once the offset is reached. Until then, the stoploss remains at the configured`stoploss`. + +## Changing stoploss on open trades + +A stoploss on an open trade can be changed by changing the value in the configuration or strategy and use the `/reload_conf` command (alternatively, completely stopping and restarting the bot also works). + +The new stoploss value will be applied to open trades (and corresponding log-messages will be generated). + +### Limitations + +Stoploss values cannot be changed if `trailing_stop` is enabled and the stoploss has already been adjusted, or if [Edge](edge.md) is enabled (since Edge would recalculate stoploss based on the current market situation). diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 1256f4a32..85d8104b0 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -223,7 +223,7 @@ For the full documentation on stoploss features, look at the dedicated [stoploss If your exchange supports it, it's recommended to also set `"stoploss_on_exchange"` in the order dict, so your stoploss is on the exchange and cannot be missed for network-problems (or other problems). -For more information on order_types please look to [here](configuration.md#understand-order_types). +For more information on order_types please look [here](configuration.md#understand-order_types). ### Ticker interval From 51aa469f67e0ed3c7504e444063032e6d60f9553 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 May 2019 20:13:01 +0200 Subject: [PATCH 08/70] Cleanups --- docs/stoploss.md | 1 - freqtrade/freqtradebot.py | 1 - freqtrade/persistence.py | 1 - freqtrade/tests/test_freqtradebot.py | 2 +- freqtrade/tests/test_persistence.py | 1 - 5 files changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/stoploss.md b/docs/stoploss.md index 975c2aeb5..f5e2f8df6 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -7,7 +7,6 @@ Most of the strategy files already include the optimal `stoploss` value. This parameter is optional. If you use it in the configuration file, it will take over the `stoploss` value from the strategy file. - ## Stop Loss support At this stage the bot contains the following stoploss support modes: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0121512ee..81c9dc5d3 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -93,7 +93,6 @@ class FreqtradeBot(object): """ Called on startup and after reloading the bot - triggers notifications and performs startup tasks - : return: None """ self.rpc.startup_messages(self.config, self.pairlists) if not self.edge: diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 3fc9a189e..ed09f6f22 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -436,7 +436,6 @@ class Trade(_DECL_BASE): # Stoploss value got changed logger.info(f"Stoploss for {trade} needs adjustment.") - logger.info(f"Stoploss: {trade.initial_stop_loss_pct}: {desired_stoploss}") # Force reset of stoploss trade.stop_loss = None trade.adjust_stop_loss(trade.open_rate, desired_stoploss) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 48eb51b54..2587b1b36 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -114,7 +114,7 @@ def test_cleanup(mocker, default_conf, caplog) -> None: def test_worker_running(mocker, default_conf, caplog) -> None: mock_throttle = MagicMock() mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) - mocker.patch('freqtrade.persistence.Trade.adjust_initial_stoploss', MagicMock()) + mocker.patch('freqtrade.persistence.Trade.stoploss_reinitialization', MagicMock()) worker = get_patched_worker(mocker, default_conf) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 93bad0797..3312bc21d 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -822,7 +822,6 @@ def test_stoploss_reinitialization(default_conf, fee): assert trade_adj.initial_stop_loss == 0.96 assert trade_adj.initial_stop_loss_pct == -0.04 - # Trailing stoploss (move stoplos up a bit) trade.adjust_stop_loss(1.02, 0.04) assert trade_adj.stop_loss == 0.9792 From fb88953be330d3e7bbb0d8f15de39087e4737096 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 29 May 2019 21:57:14 +0300 Subject: [PATCH 09/70] refactoring download_backtest_data.py --- freqtrade/arguments.py | 47 +++++++----- freqtrade/configuration.py | 31 +++++--- freqtrade/tests/test_arguments.py | 8 +-- scripts/download_backtest_data.py | 116 +++++++++++++++++------------- 4 files changed, 119 insertions(+), 83 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index ddc0dc489..c94c02a8b 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -47,7 +47,7 @@ class Arguments(object): return self.parsed_arg - def parse_args(self) -> argparse.Namespace: + def parse_args(self, no_default_config: bool = False) -> argparse.Namespace: """ Parses given arguments and returns an argparse Namespace instance. """ @@ -55,7 +55,7 @@ class Arguments(object): # Workaround issue in argparse with action='append' and default value # (see https://bugs.python.org/issue16399) - if parsed_arg.config is None: + if parsed_arg.config is None and not no_default_config: parsed_arg.config = [constants.DEFAULT_CONFIG] return parsed_arg @@ -427,26 +427,24 @@ class Arguments(object): default=None ) - def testdata_dl_options(self) -> None: + def download_data_options(self) -> None: """ Parses given arguments for testdata download """ self.parser.add_argument( - '--pairs-file', - help='File containing a list of pairs to download.', - dest='pairs_file', - default=None, - metavar='PATH', + '-v', '--verbose', + help='Verbose mode (-vv for more, -vvv to get all messages).', + action='count', + dest='loglevel', + default=0, ) - self.parser.add_argument( - '--export', - help='Export files to given dir.', - dest='export', - default=None, - metavar='PATH', + '--logfile', + help='Log to the file specified', + dest='logfile', + type=str, + metavar='FILE' ) - self.parser.add_argument( '-c', '--config', help='Specify configuration file (default: %(default)s). ' @@ -456,7 +454,21 @@ class Arguments(object): type=str, metavar='PATH', ) - + self.parser.add_argument( + '-d', '--datadir', + help='Path to backtest data.', + dest='datadir', + default=None, + type=str, + metavar='PATH', + ) + self.parser.add_argument( + '--pairs-file', + help='File containing a list of pairs to download.', + dest='pairs_file', + default=None, + metavar='PATH', + ) self.parser.add_argument( '--days', help='Download data for given number of days.', @@ -465,7 +477,6 @@ class Arguments(object): metavar='INT', default=None ) - self.parser.add_argument( '--exchange', help='Exchange name (default: %(default)s). Only valid if no config is provided.', @@ -473,7 +484,6 @@ class Arguments(object): type=str, default='bittrex' ) - self.parser.add_argument( '-t', '--timeframes', help='Specify which tickers to download. Space separated list. \ @@ -484,7 +494,6 @@ class Arguments(object): nargs='+', dest='timeframes', ) - self.parser.add_argument( '--erase', help='Clean all existing data for the selected exchange/pairs/timeframes.', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index c19580c36..58fd1d6d9 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -122,12 +122,11 @@ class Configuration(object): return conf - def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + def _load_logging_config(self, config: Dict[str, Any]) -> None: """ - Extract information for sys.argv and load common configuration - :return: configuration as dictionary + Extract information for sys.argv and load logging configuration: + the --loglevel, --logfile options """ - # Log level if 'loglevel' in self.args and self.args.loglevel: config.update({'verbosity': self.args.loglevel}) @@ -153,6 +152,13 @@ class Configuration(object): set_loggers(config['verbosity']) logger.info('Verbosity set to %s', config['verbosity']) + def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]: + """ + Extract information for sys.argv and load common configuration + :return: configuration as dictionary + """ + self._load_logging_config(config) + # Support for sd_notify if self.args.sd_notify: config['internals'].update({'sd_notify': True}) @@ -228,6 +234,17 @@ class Configuration(object): else: logger.info(logstring.format(config[argname])) + def _load_datadir_config(self, config: Dict[str, Any]) -> None: + """ + Extract information for sys.argv and load datadir configuration: + the --datadir option + """ + if 'datadir' in self.args and self.args.datadir: + config.update({'datadir': self._create_datadir(config, self.args.datadir)}) + else: + config.update({'datadir': self._create_datadir(config, None)}) + logger.info('Using data folder: %s ...', config.get('datadir')) + def _load_optimize_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ Extract information for sys.argv and load Optimize configuration @@ -263,11 +280,7 @@ class Configuration(object): self._args_to_config(config, argname='timerange', logstring='Parameter --timerange detected: {} ...') - if 'datadir' in self.args and self.args.datadir: - config.update({'datadir': self._create_datadir(config, self.args.datadir)}) - else: - config.update({'datadir': self._create_datadir(config, None)}) - logger.info('Using data folder: %s ...', config.get('datadir')) + self._load_datadir_config(config) self._args_to_config(config, argname='refresh_pairs', logstring='Parameter -r/--refresh-pairs-cached detected ...') diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index ecd108b5e..afa42f287 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -170,18 +170,18 @@ def test_parse_args_hyperopt_custom() -> None: assert call_args.func is not None -def test_testdata_dl_options() -> None: +def test_download_data_options() -> None: args = [ '--pairs-file', 'file_with_pairs', - '--export', 'export/folder', + '--datadir', 'datadir/folder', '--days', '30', '--exchange', 'binance' ] arguments = Arguments(args, '') - arguments.testdata_dl_options() + arguments.download_data_options() args = arguments.parse_args() assert args.pairs_file == 'file_with_pairs' - assert args.export == 'export/folder' + assert args.datadir == 'datadir/folder' assert args.days == 30 assert args.exchange == 'binance' diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 42b305778..acf86d0d8 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -1,39 +1,39 @@ #!/usr/bin/env python3 """ -This script generates json data +This script generates json files with pairs history data """ +import arrow import json import sys from pathlib import Path -import arrow -from typing import Any, Dict +from typing import Any, Dict, List -from freqtrade.arguments import Arguments -from freqtrade.arguments import TimeRange -from freqtrade.exchange import Exchange +from freqtrade.arguments import Arguments, TimeRange +from freqtrade.configuration import Configuration from freqtrade.data.history import download_pair_history -from freqtrade.configuration import Configuration, set_loggers +from freqtrade.exchange import Exchange from freqtrade.misc import deep_merge_dicts import logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', -) -set_loggers(0) + +logger = logging.getLogger('download_backtest_data') DEFAULT_DL_PATH = 'user_data/data' arguments = Arguments(sys.argv[1:], 'download utility') -arguments.testdata_dl_options() -args = arguments.parse_args() +arguments.download_data_options() + +# Do not read the default config if config is not specified +# in the command line options explicitely +args = arguments.parse_args(no_default_config=True) timeframes = args.timeframes +pairs: List = [] + +configuration = Configuration(args) +config: Dict[str, Any] = {} if args.config: - configuration = Configuration(args) - - config: Dict[str, Any] = {} # Now expecting a list of config filenames here, not a string for path in args.config: print(f"Using config: {path}...") @@ -42,9 +42,19 @@ if args.config: config['stake_currency'] = '' # Ensure we do not use Exchange credentials + config['exchange']['dry_run'] = True config['exchange']['key'] = '' config['exchange']['secret'] = '' + + if args.exchange: + config['exchange']['name'] = args.exchange + + pairs = config['exchange']['pair_whitelist'] + timeframes = [config['ticker_interval']] + else: + if not args.exchange: + sys.exit("No exchange specified.") config = { 'stake_currency': '', 'dry_run': True, @@ -60,55 +70,59 @@ else: } } +configuration._load_logging_config(config) +configuration._load_datadir_config(config) -dl_path = Path(DEFAULT_DL_PATH).joinpath(config['exchange']['name']) -if args.export: - dl_path = Path(args.export) - -if not dl_path.is_dir(): - sys.exit(f'Directory {dl_path} does not exist.') +dl_path = Path(config['datadir']) pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json') -if not pairs_file.exists(): - sys.exit(f'No pairs file found with path {pairs_file}.') -with pairs_file.open() as file: - PAIRS = list(set(json.load(file))) +if not pairs or args.pairs_file: + print(f'Reading pairs file "{pairs_file}".') + # Download pairs from the pairs file if no config is specified + # or if pairs file is specified explicitely + if not pairs_file.exists(): + sys.exit(f'No pairs file found with path "{pairs_file}".') -PAIRS.sort() + with pairs_file.open() as file: + pairs = list(set(json.load(file))) + pairs.sort() timerange = TimeRange() if args.days: time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d") timerange = arguments.parse_timerange(f'{time_since}-') +print(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}') -print(f'About to download pairs: {PAIRS} to {dl_path}') - -# Init exchange -exchange = Exchange(config) pairs_not_available = [] -for pair in PAIRS: - if pair not in exchange._api.markets: - pairs_not_available.append(pair) - print(f"skipping pair {pair}") - continue - for ticker_interval in timeframes: - pair_print = pair.replace('/', '_') - filename = f'{pair_print}-{ticker_interval}.json' - dl_file = dl_path.joinpath(filename) - if args.erase and dl_file.exists(): - print(f'Deleting existing data for pair {pair}, interval {ticker_interval}') - dl_file.unlink() +try: + # Init exchange + exchange = Exchange(config) - print(f'downloading pair {pair}, interval {ticker_interval}') - download_pair_history(datadir=dl_path, exchange=exchange, - pair=pair, - ticker_interval=ticker_interval, - timerange=timerange) + for pair in pairs: + if pair not in exchange._api.markets: + pairs_not_available.append(pair) + print(f"skipping pair {pair}") + continue + for ticker_interval in timeframes: + pair_print = pair.replace('/', '_') + filename = f'{pair_print}-{ticker_interval}.json' + dl_file = dl_path.joinpath(filename) + if args.erase and dl_file.exists(): + print(f'Deleting existing data for pair {pair}, interval {ticker_interval}') + dl_file.unlink() + print(f'downloading pair {pair}, interval {ticker_interval}') + download_pair_history(datadir=dl_path, exchange=exchange, + pair=pair, ticker_interval=ticker_interval, + timerange=timerange) -if pairs_not_available: - print(f"Pairs [{','.join(pairs_not_available)}] not availble.") +except KeyboardInterrupt: + sys.exit("SIGINT received, aborting ...") + +finally: + if pairs_not_available: + print(f"Pairs [{','.join(pairs_not_available)}] not availble.") From f463817c883a8510054bbc486c5c18bfe8da9eab Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 30 May 2019 09:51:32 +0300 Subject: [PATCH 10/70] change metavar for --pairs-file --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index c94c02a8b..711975fd0 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -467,7 +467,7 @@ class Arguments(object): help='File containing a list of pairs to download.', dest='pairs_file', default=None, - metavar='PATH', + metavar='FILE', ) self.parser.add_argument( '--days', From 11f535e79f74d0fd41632768c65773595356c75d Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 30 May 2019 09:54:58 +0300 Subject: [PATCH 11/70] change prints to logging --- scripts/download_backtest_data.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index acf86d0d8..acdfb25ac 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -36,7 +36,7 @@ config: Dict[str, Any] = {} if args.config: # Now expecting a list of config filenames here, not a string for path in args.config: - print(f"Using config: {path}...") + logger.info(f"Using config: {path}...") # Merge config options, overwriting old values config = deep_merge_dicts(configuration._load_config_file(path), config) @@ -78,7 +78,7 @@ dl_path = Path(config['datadir']) pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json') if not pairs or args.pairs_file: - print(f'Reading pairs file "{pairs_file}".') + logger.info(f'Reading pairs file "{pairs_file}".') # Download pairs from the pairs file if no config is specified # or if pairs file is specified explicitely if not pairs_file.exists(): @@ -94,7 +94,7 @@ if args.days: time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d") timerange = arguments.parse_timerange(f'{time_since}-') -print(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}') +logger.info(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}') pairs_not_available = [] @@ -105,17 +105,17 @@ try: for pair in pairs: if pair not in exchange._api.markets: pairs_not_available.append(pair) - print(f"skipping pair {pair}") + logger.info(f"skipping pair {pair}") continue for ticker_interval in timeframes: pair_print = pair.replace('/', '_') filename = f'{pair_print}-{ticker_interval}.json' dl_file = dl_path.joinpath(filename) if args.erase and dl_file.exists(): - print(f'Deleting existing data for pair {pair}, interval {ticker_interval}') + logger.info(f'Deleting existing data for pair {pair}, interval {ticker_interval}') dl_file.unlink() - print(f'downloading pair {pair}, interval {ticker_interval}') + logger.info(f'downloading pair {pair}, interval {ticker_interval}') download_pair_history(datadir=dl_path, exchange=exchange, pair=pair, ticker_interval=ticker_interval, timerange=timerange) @@ -125,4 +125,4 @@ except KeyboardInterrupt: finally: if pairs_not_available: - print(f"Pairs [{','.join(pairs_not_available)}] not availble.") + logger.info(f"Pairs [{','.join(pairs_not_available)}] not availble.") From 39932627bd25d5fde92242a6e307574846e372c4 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 30 May 2019 11:03:17 +0300 Subject: [PATCH 12/70] typo in log message fixed --- scripts/download_backtest_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index acdfb25ac..b7c302437 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -125,4 +125,4 @@ except KeyboardInterrupt: finally: if pairs_not_available: - logger.info(f"Pairs [{','.join(pairs_not_available)}] not availble.") + logger.info(f"Pairs [{','.join(pairs_not_available)}] not available.") From ef15f2bdc67c65ec2d05f730006502f536a8cde4 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 30 May 2019 11:07:31 +0300 Subject: [PATCH 13/70] log messages slightly improved --- scripts/download_backtest_data.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index b7c302437..196af5bed 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -105,17 +105,18 @@ try: for pair in pairs: if pair not in exchange._api.markets: pairs_not_available.append(pair) - logger.info(f"skipping pair {pair}") + logger.info(f"Skipping pair {pair}...") continue for ticker_interval in timeframes: pair_print = pair.replace('/', '_') filename = f'{pair_print}-{ticker_interval}.json' dl_file = dl_path.joinpath(filename) if args.erase and dl_file.exists(): - logger.info(f'Deleting existing data for pair {pair}, interval {ticker_interval}') + logger.info( + f'Deleting existing data for pair {pair}, interval {ticker_interval}.') dl_file.unlink() - logger.info(f'downloading pair {pair}, interval {ticker_interval}') + logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') download_pair_history(datadir=dl_path, exchange=exchange, pair=pair, ticker_interval=ticker_interval, timerange=timerange) @@ -125,4 +126,6 @@ except KeyboardInterrupt: finally: if pairs_not_available: - logger.info(f"Pairs [{','.join(pairs_not_available)}] not available.") + logger.info( + f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {config['exchange']['name']}.") From 1add432673aa76c378bed6030ce0f53939353443 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Thu, 30 May 2019 23:00:19 +0300 Subject: [PATCH 14/70] docs adjusted --- docs/backtesting.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index a25d3c1d5..5a25bc255 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -123,11 +123,12 @@ python scripts/download_backtest_data.py --exchange binance This will download ticker data for all the currency pairs you defined in `pairs.json`. -- To use a different folder than the exchange specific default, use `--export user_data/data/some_directory`. +- To use a different folder than the exchange specific default, use `--datadir user_data/data/some_directory`. - To change the exchange used to download the tickers, use `--exchange`. Default is `bittrex`. - To use `pairs.json` from some other folder, use `--pairs-file some_other_dir/pairs.json`. - To download ticker data for only 10 days, use `--days 10`. - Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. +- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with other options. For help about backtesting usage, please refer to [Backtesting commands](#backtesting-commands). From 199426460a4b1ba6bae242eff2b7b07e9f5ddfcc Mon Sep 17 00:00:00 2001 From: Yuliyan Perfanov Date: Sun, 2 Jun 2019 13:25:09 +0300 Subject: [PATCH 15/70] implemented DataProvider.orderbook() --- freqtrade/data/dataprovider.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index df4accf93..2852cbcb0 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -85,8 +85,7 @@ class DataProvider(object): """ return latest orderbook data """ - # TODO: Implement me - pass + return self._exchange.get_order_book(pair, max) @property def runmode(self) -> RunMode: From c68fe7a6857468c8c39c4aec1110c393663b2883 Mon Sep 17 00:00:00 2001 From: Yuliyan Perfanov Date: Sun, 2 Jun 2019 13:27:44 +0300 Subject: [PATCH 16/70] example how to use best bid and ask in strategy --- user_data/strategies/test_strategy.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 66a5f8c09..7edaead14 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -253,6 +253,15 @@ class TestStrategy(IStrategy): dataframe['ha_low'] = heikinashi['low'] """ + # Retrieve best bid and best ask + # ------------------------------------ + """ + ob = self.dp.orderbook(metadata['pair'], 1) + print(ob) + dataframe['best_bid'] = ob['bids'][0][0] + dataframe['best_ask'] = ob['asks'][0][0] + """ + return dataframe def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: From 36dae7cc6c513e590ff8cbeb36be5b82b8393664 Mon Sep 17 00:00:00 2001 From: Misagh Date: Sun, 2 Jun 2019 13:27:31 +0200 Subject: [PATCH 17/70] trailing stoploss reason fixed --- freqtrade/strategy/interface.py | 5 +++-- freqtrade/tests/test_freqtradebot.py | 32 ++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index caf56f13e..db266d95f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -329,8 +329,9 @@ class IStrategy(ABC): (not self.order_types.get('stoploss_on_exchange'))): selltype = SellType.STOP_LOSS - # If Trailing stop (and max-rate did move above open rate) - if trailing_stop and trade.open_rate != trade.max_rate: + + # If initial stoploss is not the same as current one then it is trailing. + if trade.initial_stop_loss != trade.stop_loss: selltype = SellType.TRAILING_STOP_LOSS logger.debug( f"HIT STOP: current price at {current_rate:.6f}, " diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 67b05ac3e..790e90641 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -2493,9 +2493,9 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=MagicMock(return_value={ - 'bid': 0.00000102, - 'ask': 0.00000103, - 'last': 0.00000102 + 'bid': 0.00001099, + 'ask': 0.00001099, + 'last': 0.00001099 }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, @@ -2507,15 +2507,33 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trade() - trade = Trade.query.first() - trade.update(limit_buy_order) - trade.max_rate = trade.open_rate * 1.003 + assert freqtrade.handle_trade(trade) is False + + # Raise ticker above buy price + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={ + 'bid': 0.00001099 * 1.5, + 'ask': 0.00001099 * 1.5, + 'last': 0.00001099 * 1.5 + })) + + # Stoploss should be adjusted + assert freqtrade.handle_trade(trade) is False + + # Price fell + mocker.patch('freqtrade.exchange.Exchange.get_ticker', + MagicMock(return_value={ + 'bid': 0.00001099 * 1.1, + 'ask': 0.00001099 * 1.1, + 'last': 0.00001099 * 1.1 + })) + caplog.set_level(logging.DEBUG) # Sell as trailing-stop is reached assert freqtrade.handle_trade(trade) is True assert log_has( - f'HIT STOP: current price at 0.000001, stop loss is {trade.stop_loss:.6f}, ' + f'HIT STOP: current price at 0.000012, stop loss is 0.000015, ' f'initial stop loss was at 0.000010, trade opened at 0.000011', caplog.record_tuples) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value From bd8edd61fd29f304bdfdb6c03fcc81b6a45cfaf4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 3 Jun 2019 17:19:12 +0200 Subject: [PATCH 18/70] Update numpy from 1.16.3 to 1.16.4 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index da87f56d9..52442fb19 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # Load common requirements -r requirements-common.txt -numpy==1.16.3 +numpy==1.16.4 pandas==0.24.2 scipy==1.3.0 From c04a8a102469a099230e09c85b8fb4e48798c5d0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 3 Jun 2019 17:19:13 +0200 Subject: [PATCH 19/70] Update ccxt from 1.18.578 to 1.18.615 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 434944aad..7620ddada 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.578 +ccxt==1.18.615 SQLAlchemy==1.3.3 python-telegram-bot==11.1.0 arrow==0.13.2 From 51113dae0e2d21fee9095ddaa5b03ddf931ff86d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 3 Jun 2019 17:19:16 +0200 Subject: [PATCH 20/70] Update sqlalchemy from 1.3.3 to 1.3.4 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 7620ddada..ee8b08b75 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,7 +1,7 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs ccxt==1.18.615 -SQLAlchemy==1.3.3 +SQLAlchemy==1.3.4 python-telegram-bot==11.1.0 arrow==0.13.2 cachetools==3.1.1 From 4ef8a74977acf7fb55a8350269f2be00e7a616fd Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 3 Jun 2019 17:19:19 +0200 Subject: [PATCH 21/70] Update arrow from 0.13.2 to 0.14.1 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index ee8b08b75..bf21c6e59 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -3,7 +3,7 @@ ccxt==1.18.615 SQLAlchemy==1.3.4 python-telegram-bot==11.1.0 -arrow==0.13.2 +arrow==0.14.1 cachetools==3.1.1 requests==2.22.0 urllib3==1.24.2 # pyup: ignore From 3c1ae07f92d1399cbc336f37d32db1afb9801050 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 3 Jun 2019 17:19:20 +0200 Subject: [PATCH 22/70] Update flask from 1.0.2 to 1.0.3 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index bf21c6e59..9e854e4af 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -29,4 +29,4 @@ python-rapidjson==0.7.1 sdnotify==0.3.2 # Api server -flask==1.0.2 +flask==1.0.3 From a132517f0abeb60071608ba39b405cdaa7838f60 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 3 Jun 2019 17:19:24 +0200 Subject: [PATCH 23/70] Update pytest from 4.5.0 to 4.6.1 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index fa52a4869..531d99940 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ flake8==3.7.7 flake8-type-annotations==0.1.0 flake8-tidy-imports==2.0.0 -pytest==4.5.0 +pytest==4.6.1 pytest-mock==1.10.4 pytest-asyncio==0.10.0 pytest-cov==2.7.1 From f75e97e9b0e15cc114101051ab4d57a038b7ddc3 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 3 Jun 2019 17:19:25 +0200 Subject: [PATCH 24/70] Update coveralls from 1.7.0 to 1.8.0 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 531d99940..effa714e9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,5 +8,5 @@ pytest==4.6.1 pytest-mock==1.10.4 pytest-asyncio==0.10.0 pytest-cov==2.7.1 -coveralls==1.7.0 +coveralls==1.8.0 mypy==0.701 From 7134273918e927d010283a4626380c7cdcf532c5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 3 Jun 2019 17:19:26 +0200 Subject: [PATCH 25/70] Update plotly from 3.9.0 to 3.10.0 --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index 23daee258..d4e4fc165 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==3.9.0 +plotly==3.10.0 From 2e6ded06a91d8b51930d25ec9b77380c115b15a2 Mon Sep 17 00:00:00 2001 From: Yuliyan Perfanov Date: Thu, 6 Jun 2019 18:25:58 +0300 Subject: [PATCH 26/70] removed redundant print() --- user_data/strategies/test_strategy.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 7edaead14..6b7770a8f 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -253,15 +253,16 @@ class TestStrategy(IStrategy): dataframe['ha_low'] = heikinashi['low'] """ - # Retrieve best bid and best ask + # Retrieve best bid and best ask # ------------------------------------ """ - ob = self.dp.orderbook(metadata['pair'], 1) - print(ob) - dataframe['best_bid'] = ob['bids'][0][0] - dataframe['best_ask'] = ob['asks'][0][0] + # first check if dataprovider is available + if self.dp: + ob = self.dp.orderbook(metadata['pair'], 1) + dataframe['best_bid'] = ob['bids'][0][0] + dataframe['best_ask'] = ob['asks'][0][0] """ - + return dataframe def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: From a9ed5da369d606da23223b16bf7fcc834bb13949 Mon Sep 17 00:00:00 2001 From: Yuliyan Perfanov Date: Thu, 6 Jun 2019 18:48:26 +0300 Subject: [PATCH 27/70] added doc for DataProvider.orderbook() --- docs/strategy-customization.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 51540f690..b44775dfb 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -298,6 +298,18 @@ if self.dp: !!! Warning Warning in hyperopt This option cannot currently be used during hyperopt. +#### Orderbook + +``` python +if self.dp: + if self.dp.runmode in ('live', 'dry_run'): + ob = self.dp.orderbook(metadata['pair'], 1) + dataframe['best_bid'] = ob['bids'][0][0] + dataframe['best_ask'] = ob['asks'][0][0] +``` +!Warning The order book is not part of the historic data which means backtesting and hyperopt will not work if this + method is used. + #### Available Pairs ``` python @@ -306,6 +318,7 @@ if self.dp: print(f"available {pair}, {ticker}") ``` + #### Get data for non-tradeable pairs Data for additional, informative pairs (reference pairs) can be beneficial for some strategies. From f9fe2663644cd003224919540cc95bddbee8fc36 Mon Sep 17 00:00:00 2001 From: Yuliyan Perfanov Date: Thu, 6 Jun 2019 18:52:14 +0300 Subject: [PATCH 28/70] check for runmode before retrieving the orderbook --- user_data/strategies/test_strategy.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 6b7770a8f..2415e43eb 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -258,9 +258,10 @@ class TestStrategy(IStrategy): """ # first check if dataprovider is available if self.dp: - ob = self.dp.orderbook(metadata['pair'], 1) - dataframe['best_bid'] = ob['bids'][0][0] - dataframe['best_ask'] = ob['asks'][0][0] + if self.dp.runmode in ('live', 'dry_run'): + ob = self.dp.orderbook(metadata['pair'], 1) + dataframe['best_bid'] = ob['bids'][0][0] + dataframe['best_ask'] = ob['asks'][0][0] """ return dataframe From 5273540a93fd8ce33b998a2674024a0a1d133702 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 8 Jun 2019 19:32:31 +0200 Subject: [PATCH 29/70] Fix test failure (double-trailing newlines are removed now) --- freqtrade/tests/optimize/test_hyperopt.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index b41f8ac36..a51d74dbb 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -372,20 +372,21 @@ def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: ) patch_exchange(mocker) - default_conf.update({'config': 'config.json.example'}) - default_conf.update({'epochs': 1}) - default_conf.update({'timerange': None}) - default_conf.update({'spaces': 'all'}) - default_conf.update({'hyperopt_jobs': 1}) + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'all', + 'hyperopt_jobs': 1, }) hyperopt = Hyperopt(default_conf) hyperopt.strategy.tickerdata_to_dataframe = MagicMock() hyperopt.start() parallel.assert_called_once() - - assert 'Best result:\nfoo result\nwith values:\n\n' in caplog.text + assert log_has('Best result:\nfoo result\nwith values:\n', caplog.record_tuples) assert dumper.called + # Should be called twice, once for tickerdata, once to save evaluations + assert dumper.call_count == 2 def test_format_results(hyperopt): From adc12ed043320a9aee2884cdfad6aa5e86cb3069 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 8 Jun 2019 20:26:25 +0200 Subject: [PATCH 30/70] Fix new test after develop merge --- freqtrade/tests/test_persistence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/test_persistence.py b/freqtrade/tests/test_persistence.py index 57f054dee..bb00fa8f4 100644 --- a/freqtrade/tests/test_persistence.py +++ b/freqtrade/tests/test_persistence.py @@ -780,7 +780,7 @@ def test_to_json(default_conf, fee): def test_stoploss_reinitialization(default_conf, fee): - init(default_conf) + init(default_conf['db_url']) trade = Trade( pair='ETH/BTC', stake_amount=0.001, From d7c63347e12f41a99e3caaed498669f0230ea16e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 13:19:01 +0200 Subject: [PATCH 31/70] Use kwarg for parse_ticker_dataframe --- freqtrade/data/history.py | 2 +- freqtrade/tests/conftest.py | 4 ++-- freqtrade/tests/strategy/test_interface.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 2dacce8c6..72f7111f6 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -106,7 +106,7 @@ def load_pair_history(pair: str, logger.warning('Missing data at end for pair %s, data ends at %s', pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - return parse_ticker_dataframe(pairdata, ticker_interval, fill_up_missing) + return parse_ticker_dataframe(pairdata, ticker_interval, fill_missing=fill_up_missing) else: logger.warning( f'No history data for pair: "{pair}", interval: {ticker_interval}. ' diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index a907b33ed..dcc69fcb1 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -649,7 +649,7 @@ def ticker_history_list(): @pytest.fixture def ticker_history(ticker_history_list): - return parse_ticker_dataframe(ticker_history_list, "5m", True) + return parse_ticker_dataframe(ticker_history_list, "5m", fill_missing=True) @pytest.fixture @@ -854,7 +854,7 @@ def tickers(): @pytest.fixture def result(): with open('freqtrade/tests/testdata/UNITTEST_BTC-1m.json') as data_file: - return parse_ticker_dataframe(json.load(data_file), '1m', True) + return parse_ticker_dataframe(json.load(data_file), '1m', fill_missing=True) # FIX: # Create an fixture/function diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index d6ef0c8e7..e384003dc 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -111,7 +111,7 @@ def test_tickerdata_to_dataframe(default_conf) -> None: timerange = TimeRange(None, 'line', 0, -100) tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', True)} + tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', fill_missing=True)} data = strategy.tickerdata_to_dataframe(tickerlist) assert len(data['UNITTEST/BTC']) == 102 # partial candle was removed From 9c497bf15c597921bb1c5509236f50414bbc4df4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:04:19 +0200 Subject: [PATCH 32/70] Improve docstring for deep_merge_dicts --- freqtrade/misc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 9d37214e4..460e20e91 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -117,6 +117,8 @@ def format_ms_time(date: int) -> str: def deep_merge_dicts(source, destination): """ + Values from Source override destination, destination is returned (and modified!!) + Sample: >>> a = { 'first' : { 'rows' : { 'pass' : 'dog', 'number' : '1' } } } >>> b = { 'first' : { 'rows' : { 'fail' : 'cat', 'number' : '5' } } } >>> merge(b, a) == { 'first' : { 'rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } } From 7108a2e57d557b4fc2580ec0defb2693c7e96164 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:05:36 +0200 Subject: [PATCH 33/70] Add deep_merge for _ft_has and test --- freqtrade/exchange/exchange.py | 31 +++++++++++++++-------- freqtrade/tests/exchange/test_exchange.py | 27 ++++++++++++++++++++ 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 72a0efb1f..07a5e9037 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2,23 +2,24 @@ """ Cryptocurrency Exchanges support """ -import logging +import asyncio import inspect -from random import randint -from typing import List, Dict, Tuple, Any, Optional +import logging +from copy import deepcopy from datetime import datetime -from math import floor, ceil +from math import ceil, floor +from random import randint +from typing import Any, Dict, List, Optional, Tuple import arrow -import asyncio import ccxt import ccxt.async_support as ccxt_async from pandas import DataFrame -from freqtrade import (constants, DependencyException, OperationalException, - TemporaryError, InvalidOrderException) +from freqtrade import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError, constants) from freqtrade.data.converter import parse_ticker_dataframe - +from freqtrade.misc import deep_merge_dicts logger = logging.getLogger(__name__) @@ -68,12 +69,13 @@ class Exchange(object): _params: Dict = {} # Dict to specify which options each exchange implements - # TODO: this should be merged with attributes from subclasses - # To avoid having to copy/paste this to all subclasses. - _ft_has: Dict = { + # This defines defaults, which can be selectively overridden by subclasses using _ft_has + # or by specifying them in the configuration. + _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], } + _ft_has: Dict = {} def __init__(self, config: dict) -> None: """ @@ -100,6 +102,13 @@ class Exchange(object): logger.info('Instance is running with dry_run enabled') exchange_config = config['exchange'] + + # Deep merge ft_has with default ft_has options + self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default)) + if exchange_config.get("_ft_has_params"): + self._ft_has = deep_merge_dicts(exchange_config.get("_ft_has_params"), + self._ft_has) + self._api: ccxt.Exchange = self._init_ccxt( exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config')) self._api_async: ccxt_async.Exchange = self._init_ccxt( diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index fda9c8241..f0dc96626 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -1435,3 +1435,30 @@ def test_stoploss_limit_order_dry_run(default_conf, mocker): assert order['type'] == order_type assert order['price'] == 220 assert order['amount'] == 1 + + +def test_merge_ft_has_dict(default_conf, mocker): + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock())) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + ex = Exchange(default_conf) + assert ex._ft_has == Exchange._ft_has_default + + ex = Kraken(default_conf) + assert ex._ft_has == Exchange._ft_has_default + + # Binance defines different values + ex = Binance(default_conf) + assert ex._ft_has != Exchange._ft_has_default + assert ex._ft_has['stoploss_on_exchange'] + assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc'] + + conf = copy.deepcopy(default_conf) + conf['exchange']['_ft_has_params'] = {"DeadBeef": 20, + "stoploss_on_exchange": False} + # Use settings from configuration (overriding stoploss_on_exchange) + ex = Binance(conf) + assert ex._ft_has != Exchange._ft_has_default + assert not ex._ft_has['stoploss_on_exchange'] + assert ex._ft_has['DeadBeef'] == 20 From 3fe5388d4cdce72a03f7420a5585eaf8edddbd21 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:13:03 +0200 Subject: [PATCH 34/70] Document _ft_has_params override --- docs/configuration.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index df116b3c2..bbadb87e8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -301,8 +301,24 @@ This configuration enables binance, as well as rate limiting to avoid bans from !!! Note Optimal settings for rate limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings. - We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. + We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. +#### Advanced FreqTrade Exchange configuration + +Advanced options can be configured using the `_ft_has_params` setting, which will override Defaults and exchange-specific behaviours. + +Available options are listed in the exchange-class as `_ft_has_default`. + +For example, to test the order type `FOK` with Kraken: + +```json +"exchange": { + "name": "kraken", + "_ft_has_params": {"order_time_in_force": ["gtc", "fok"]} +``` + +!!! Warning + Please make sure to fully understand the impacts of these settings before modifying them. ### What values can be used for fiat_display_currency? From fdbbefdddd8442f1439256b84ddb24f804e1490c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:35:58 +0200 Subject: [PATCH 35/70] Make drop_incomplete optional --- freqtrade/data/converter.py | 12 ++++++++---- freqtrade/exchange/exchange.py | 8 +++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 77a3447da..dc566070d 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -10,14 +10,16 @@ from pandas import DataFrame, to_datetime logger = logging.getLogger(__name__) -def parse_ticker_dataframe(ticker: list, ticker_interval: str, - fill_missing: bool = True) -> DataFrame: +def parse_ticker_dataframe(ticker: list, ticker_interval: str, *, + fill_missing: bool = True, + drop_incomplete: bool = True) -> DataFrame: """ Converts a ticker-list (format ccxt.fetch_ohlcv) to a Dataframe :param ticker: ticker list, as returned by exchange.async_get_candle_history :param ticker_interval: ticker_interval (e.g. 5m). Used to fill up eventual missing data :param fill_missing: fill up missing candles with 0 candles (see ohlcv_fill_up_missing_data for details) + :param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete :return: DataFrame """ logger.debug("Parsing tickerlist to dataframe") @@ -43,8 +45,10 @@ def parse_ticker_dataframe(ticker: list, ticker_interval: str, 'close': 'last', 'volume': 'max', }) - frame.drop(frame.tail(1).index, inplace=True) # eliminate partial candle - logger.debug('Dropping last candle') + # eliminate partial candle + if drop_incomplete: + frame.drop(frame.tail(1).index, inplace=True) + logger.debug('Dropping last candle') if fill_missing: return ohlcv_fill_up_missing_data(frame, ticker_interval) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 07a5e9037..401a3571f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -74,6 +74,7 @@ class Exchange(object): _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], + "ohlcv_partial_candle": True, } _ft_has: Dict = {} @@ -108,7 +109,12 @@ class Exchange(object): if exchange_config.get("_ft_has_params"): self._ft_has = deep_merge_dicts(exchange_config.get("_ft_has_params"), self._ft_has) + logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has) + # Assign this directly for easy access + self._drop_incomplete = self._ft_has['ohlcv_partial_candle'] + + # Initialize ccxt objects self._api: ccxt.Exchange = self._init_ccxt( exchange_config, ccxt_kwargs=exchange_config.get('ccxt_config')) self._api_async: ccxt_async.Exchange = self._init_ccxt( @@ -575,7 +581,7 @@ class Exchange(object): self._pairs_last_refresh_time[(pair, ticker_interval)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache self._klines[(pair, ticker_interval)] = parse_ticker_dataframe( - ticks, ticker_interval, fill_missing=True) + ticks, ticker_interval, fill_missing=True, drop_incomplete=self._drop_incomplete) return tickers def _now_is_time_to_refresh(self, pair: str, ticker_interval: str) -> bool: From 6ad94684d501af503cf070edb714e21f54ce2886 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:36:08 +0200 Subject: [PATCH 36/70] Add WIP document of steps to test a new exchange --- docs/developer.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/developer.md b/docs/developer.md index e7f79bc1c..ccd5cdd8f 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -81,6 +81,50 @@ Please also run `self._validate_whitelist(pairs)` and to check and remove pairs This is a simple method used by `VolumePairList` - however serves as a good example. It implements caching (`@cached(TTLCache(maxsize=1, ttl=1800))`) as well as a configuration option to allow different (but similar) strategies to work with the same PairListProvider. +## Implement a new Exchange (WIP) + +!!! Note + This section is a Work in Progress and is not a complete guide on how to test a new exchange with FreqTrade. + +Most exchanges supported by CCXT should work out of the box. + +### Stoploss On Exchange + +Check if the new exchange supports Stoploss on Exchange orders through their API. + +Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselfs. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects. + +### Incomplete candles + +While fetching OHLCV data, we're may end up getting incomplete candles (Depending on the exchange). +To demonstrate this, we'll use daily candles (`"1d"`) to keep things simple. +We query the api (`ct.fetch_ohlcv()`) for the timeframe and look at the date of the last entry. If this entry changes or shows the date of a "incomplete" candle, then we should drop this since having incomplete candles is problematic because indicators assume that only complete candles are passed to them, and will generate a lot of false buy signals. By default, we're therefore removing the last candle assuming it's incomplete. + +To check how the new exchange behaves, you can use the following snippet: + +``` python +import ccxt +from datetime import datetime +from freqtrade.data.converter import parse_ticker_dataframe +ct = ccxt.binance() +timeframe = "1d" +raw = ct.fetch_ohlcv(pair, timeframe=timeframe) + +# convert to dataframe +df1 = parse_ticker_dataframe(raw, timeframe, drop_incomplete=False) + +print(df1["date"].tail(1)) +print(datetime.utcnow()) +``` + +``` output +19 2019-06-08 00:00:00+00:00 +2019-06-09 12:30:27.873327 +``` + +The output will show the last entry from the Exchange as well as the current UTC date. +If the day shows the same day, then the last candle can be assumed as incomplete and should be dropped (leave the setting `"ohlcv_partial_candle"` from the exchange-class untouched / True). Otherwise, set `"ohlcv_partial_candle"` to `False` to not drop Candles (shown in the example above). + ## Creating a release This part of the documentation is aimed at maintainers, and shows how to create a release. From ce317b62f9b3d523ccaa9e8474f99be2c36966a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:40:45 +0200 Subject: [PATCH 37/70] Add docstrings to load_pair_history --- freqtrade/data/history.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 72f7111f6..67f942119 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -81,10 +81,20 @@ def load_pair_history(pair: str, timerange: TimeRange = TimeRange(None, None, 0, 0), refresh_pairs: bool = False, exchange: Optional[Exchange] = None, - fill_up_missing: bool = True + fill_up_missing: bool = True, + drop_incomplete: bool = True ) -> DataFrame: """ Loads cached ticker history for the given pair. + :param pair: Pair to load data for + :param ticker_interval: Ticker-interval (e.g. "5m") + :param datadir: Path to the data storage location. + :param timerange: Limit data to be loaded to this timerange + :param refresh_pairs: Refresh pairs from exchange. + (Note: Requires exchange to be passed as well.) + :param exchange: Exchange object (needed when using "refresh_pairs") + :param fill_up_missing: Fill missing values with "No action"-candles + :param drop_incomplete: Drop last candle assuming it may be incomplete. :return: DataFrame with ohlcv data """ @@ -106,7 +116,9 @@ def load_pair_history(pair: str, logger.warning('Missing data at end for pair %s, data ends at %s', pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - return parse_ticker_dataframe(pairdata, ticker_interval, fill_missing=fill_up_missing) + return parse_ticker_dataframe(pairdata, ticker_interval, + fill_missing=fill_up_missing, + drop_incomplete=drop_incomplete) else: logger.warning( f'No history data for pair: "{pair}", interval: {ticker_interval}. ' From 3380543878f664089038357d7554d947cafcfd4e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:51:58 +0200 Subject: [PATCH 38/70] Add test for drop_incomplete option --- freqtrade/tests/data/test_converter.py | 47 ++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/freqtrade/tests/data/test_converter.py b/freqtrade/tests/data/test_converter.py index 4c8de575d..8a0761f1c 100644 --- a/freqtrade/tests/data/test_converter.py +++ b/freqtrade/tests/data/test_converter.py @@ -96,3 +96,50 @@ def test_ohlcv_fill_up_missing_data2(caplog): assert log_has(f"Missing data fillup: before: {len(data)} - after: {len(data2)}", caplog.record_tuples) + + +def test_ohlcv_drop_incomplete(caplog): + ticker_interval = '1d' + ticks = [[ + 1559750400000, # 2019-06-04 + 8.794e-05, # open + 8.948e-05, # high + 8.794e-05, # low + 8.88e-05, # close + 2255, # volume (in quote currency) + ], + [ + 1559836800000, # 2019-06-05 + 8.88e-05, + 8.942e-05, + 8.88e-05, + 8.893e-05, + 9911, + ], + [ + 1559923200000, # 2019-06-06 + 8.891e-05, + 8.893e-05, + 8.875e-05, + 8.877e-05, + 2251 + ], + [ + 1560009600000, # 2019-06-07 + 8.877e-05, + 8.883e-05, + 8.895e-05, + 8.817e-05, + 123551 + ] + ] + caplog.set_level(logging.DEBUG) + data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False, drop_incomplete=False) + assert len(data) == 4 + assert not log_has("Dropping last candle", caplog.record_tuples) + + # Drop last candle + data = parse_ticker_dataframe(ticks, ticker_interval, fill_missing=False, drop_incomplete=True) + assert len(data) == 3 + + assert log_has("Dropping last candle", caplog.record_tuples) From 9f2e0b11d19772cfbe165ad6f8e67099ab98c175 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 14:52:17 +0200 Subject: [PATCH 39/70] Parametrize ohlcv_candle_limit (per call) --- docs/configuration.md | 7 +++++-- freqtrade/exchange/exchange.py | 11 ++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index bbadb87e8..98953d73f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -309,12 +309,15 @@ Advanced options can be configured using the `_ft_has_params` setting, which wil Available options are listed in the exchange-class as `_ft_has_default`. -For example, to test the order type `FOK` with Kraken: +For example, to test the order type `FOK` with Kraken, and modify candle_limit to 200 (so you only get 200 candles per call): ```json "exchange": { "name": "kraken", - "_ft_has_params": {"order_time_in_force": ["gtc", "fok"]} + "_ft_has_params": { + "order_time_in_force": ["gtc", "fok"], + "ohlcv_candle_limit": 200 + } ``` !!! Warning diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 401a3571f..ea6996efb 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -74,6 +74,7 @@ class Exchange(object): _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], + "ohlcv_candle_limit": 500, "ohlcv_partial_candle": True, } _ft_has: Dict = {} @@ -112,7 +113,8 @@ class Exchange(object): logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has) # Assign this directly for easy access - self._drop_incomplete = self._ft_has['ohlcv_partial_candle'] + self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit'] + self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle'] # Initialize ccxt objects self._api: ccxt.Exchange = self._init_ccxt( @@ -521,10 +523,8 @@ class Exchange(object): async def _async_get_history(self, pair: str, ticker_interval: str, since_ms: int) -> List: - # Assume exchange returns 500 candles - _LIMIT = 500 - one_call = timeframe_to_msecs(ticker_interval) * _LIMIT + one_call = timeframe_to_msecs(ticker_interval) * self._ohlcv_candle_limit logger.debug( "one_call: %s msecs (%s)", one_call, @@ -581,7 +581,8 @@ class Exchange(object): self._pairs_last_refresh_time[(pair, ticker_interval)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache self._klines[(pair, ticker_interval)] = parse_ticker_dataframe( - ticks, ticker_interval, fill_missing=True, drop_incomplete=self._drop_incomplete) + ticks, ticker_interval, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) return tickers def _now_is_time_to_refresh(self, pair: str, ticker_interval: str) -> bool: From 792390e8159a7f6f6e211f0a10146de4cc3741f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Jun 2019 15:03:26 +0200 Subject: [PATCH 40/70] Add missing parameter for exchange-verify snippet --- docs/developer.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/developer.md b/docs/developer.md index ccd5cdd8f..6ecb7f156 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -108,6 +108,7 @@ from datetime import datetime from freqtrade.data.converter import parse_ticker_dataframe ct = ccxt.binance() timeframe = "1d" +pair = "XLM/BTC" # Make sure to use a pair that exists on that exchange! raw = ct.fetch_ohlcv(pair, timeframe=timeframe) # convert to dataframe From 90b0f1daa860fc27d2edb37ed3fa2eefddd5f0f9 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 10 Jun 2019 02:08:54 +0300 Subject: [PATCH 41/70] minor optimize cleanup --- freqtrade/edge/__init__.py | 12 ++++-------- freqtrade/optimize/backtesting.py | 9 +++------ freqtrade/optimize/hyperopt.py | 5 +++-- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 3ddff4772..4bc8bb493 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -46,10 +46,6 @@ class Edge(): self.config = config self.exchange = exchange self.strategy = strategy - self.ticker_interval = self.strategy.ticker_interval - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.advise_sell = self.strategy.advise_sell - self.advise_buy = self.strategy.advise_buy self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs @@ -100,7 +96,7 @@ class Edge(): data = history.load_data( datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, pairs=pairs, - ticker_interval=self.ticker_interval, + ticker_interval=self.strategy.ticker_interval, refresh_pairs=self._refresh_pairs, exchange=self.exchange, timerange=self._timerange @@ -112,7 +108,7 @@ class Edge(): logger.critical("No data found. Edge is stopped ...") return False - preprocessed = self.tickerdata_to_dataframe(data) + preprocessed = self.strategy.tickerdata_to_dataframe(data) # Print timeframe min_date, max_date = history.get_timeframe(preprocessed) @@ -130,8 +126,8 @@ class Edge(): pair_data = pair_data.sort_values(by=['date']) pair_data = pair_data.reset_index(drop=True) - ticker_data = self.advise_sell( - self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() + ticker_data = self.strategy.advise_sell( + self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 76c6556fa..47933668c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -72,18 +72,16 @@ class Backtesting(object): IStrategy.dp = self.dataprovider if self.config.get('strategy_list', None): - # Force one interval - self.ticker_interval = str(self.config.get('ticker_interval')) - self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval) for strat in list(self.config['strategy_list']): stratconf = deepcopy(self.config) stratconf['strategy'] = strat self.strategylist.append(StrategyResolver(stratconf).strategy) else: - # only one strategy + # No strategy list specified, only one strategy self.strategylist.append(StrategyResolver(self.config).strategy) - # Load one strategy + + # Load one (first) strategy self._set_strategy(self.strategylist[0]) def _set_strategy(self, strategy): @@ -94,7 +92,6 @@ class Backtesting(object): self.ticker_interval = self.config.get('ticker_interval') self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval) - self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe self.advise_buy = strategy.advise_buy self.advise_sell = strategy.advise_sell # Set stoploss_on_exchange to false for backtesting, diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d19d54031..28b9ce789 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -45,7 +45,6 @@ class Hyperopt(Backtesting): """ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) - self.config = config self.custom_hyperopt = HyperOptResolver(self.config).hyperopt # set TARGET_TRADES to suit your number concurrent trades so its realistic @@ -296,7 +295,9 @@ class Hyperopt(Backtesting): self.strategy.advise_indicators = \ self.custom_hyperopt.populate_indicators # type: ignore - dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) + preprocessed = self.strategy.tickerdata_to_dataframe(data) + + dump(preprocessed, TICKERDATA_PICKLE) # We don't need exchange instance anymore while running hyperopt self.exchange = None # type: ignore From 4dc3a0ca1d3d8900bc78cca3ea31a4ca3f2e32b5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 10 Jun 2019 16:20:19 +0200 Subject: [PATCH 42/70] Small cleanup to reduce dict lookups during backtesting/hyperopt --- freqtrade/optimize/backtesting.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 47933668c..6cc78ad2b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -232,10 +232,9 @@ class Backtesting(object): def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, - partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[BacktestResult]: + partial_ticker: List, trade_count_lock: Dict, + stake_amount: float, max_open_trades: int) -> Optional[BacktestResult]: - stake_amount = args['stake_amount'] - max_open_trades = args.get('max_open_trades', 0) trade = Trade( open_rate=buy_row.open, open_date=buy_row.date, @@ -251,8 +250,7 @@ class Backtesting(object): # Increase trade_count_lock for every iteration trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1 - buy_signal = sell_row.buy - sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, + sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, sell_row.buy, sell_row.sell, low=sell_row.low, high=sell_row.high) if sell.sell_flag: @@ -325,6 +323,7 @@ class Backtesting(object): :return: DataFrame """ processed = args['processed'] + stake_amount = args['stake_amount'] max_open_trades = args.get('max_open_trades', 0) position_stacking = args.get('position_stacking', False) start_date = args['start_date'] @@ -375,7 +374,8 @@ class Backtesting(object): trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 trade_entry = self._get_sell_trade_entry(pair, row, ticker[pair][indexes[pair]:], - trade_count_lock, args) + trade_count_lock, stake_amount, + max_open_trades) if trade_entry: lock_pair_until[pair] = trade_entry.close_time From 5c5b0effc18275bbae44c65fe9f62a6d4dd11f3a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 10 Jun 2019 15:19:05 +0000 Subject: [PATCH 43/70] Update ccxt from 1.18.615 to 1.18.667 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 9e854e4af..d24ac4336 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.615 +ccxt==1.18.667 SQLAlchemy==1.3.4 python-telegram-bot==11.1.0 arrow==0.14.1 From 9961c0e15b6ccab9edd1f202be314c387aaba2f2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 10 Jun 2019 15:19:06 +0000 Subject: [PATCH 44/70] Update arrow from 0.14.1 to 0.14.2 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index d24ac4336..5100f3235 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -3,7 +3,7 @@ ccxt==1.18.667 SQLAlchemy==1.3.4 python-telegram-bot==11.1.0 -arrow==0.14.1 +arrow==0.14.2 cachetools==3.1.1 requests==2.22.0 urllib3==1.24.2 # pyup: ignore From 1a41d4e6cd5f1cf74dc79300e2e8c1e471651c7a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 10 Jun 2019 15:19:08 +0000 Subject: [PATCH 45/70] Update python-rapidjson from 0.7.1 to 0.7.2 --- requirements-common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-common.txt b/requirements-common.txt index 5100f3235..8b44e9d28 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -23,7 +23,7 @@ filelock==3.0.12 py_find_1st==1.1.3 #Load ticker files 30% faster -python-rapidjson==0.7.1 +python-rapidjson==0.7.2 # Notify systemd sdnotify==0.3.2 From 6636f0c71bc4097b2d2afc3b1c96b8b9402b6b91 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 10 Jun 2019 15:19:09 +0000 Subject: [PATCH 46/70] Update pytest from 4.6.1 to 4.6.2 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index effa714e9..315033847 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ flake8==3.7.7 flake8-type-annotations==0.1.0 flake8-tidy-imports==2.0.0 -pytest==4.6.1 +pytest==4.6.2 pytest-mock==1.10.4 pytest-asyncio==0.10.0 pytest-cov==2.7.1 From dc0326db2737ba1bc213bab7f76bbffcc922c810 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 11 Jun 2019 10:09:30 +0300 Subject: [PATCH 47/70] fix handling --exchange --- freqtrade/arguments.py | 8 +------- scripts/download_backtest_data.py | 17 +++++++++++------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 711975fd0..0d4288ef3 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -443,7 +443,7 @@ class Arguments(object): help='Log to the file specified', dest='logfile', type=str, - metavar='FILE' + metavar='FILE', ) self.parser.add_argument( '-c', '--config', @@ -458,15 +458,12 @@ class Arguments(object): '-d', '--datadir', help='Path to backtest data.', dest='datadir', - default=None, - type=str, metavar='PATH', ) self.parser.add_argument( '--pairs-file', help='File containing a list of pairs to download.', dest='pairs_file', - default=None, metavar='FILE', ) self.parser.add_argument( @@ -475,14 +472,11 @@ class Arguments(object): dest='days', type=int, metavar='INT', - default=None ) self.parser.add_argument( '--exchange', help='Exchange name (default: %(default)s). Only valid if no config is provided.', dest='exchange', - type=str, - default='bittrex' ) self.parser.add_argument( '-t', '--timeframes', diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 196af5bed..278879fb0 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -27,6 +27,9 @@ arguments.download_data_options() # in the command line options explicitely args = arguments.parse_args(no_default_config=True) +# Use bittrex as default exchange +exchange_name = args.exchange or 'bittrex' + timeframes = args.timeframes pairs: List = [] @@ -46,20 +49,15 @@ if args.config: config['exchange']['key'] = '' config['exchange']['secret'] = '' - if args.exchange: - config['exchange']['name'] = args.exchange - pairs = config['exchange']['pair_whitelist'] timeframes = [config['ticker_interval']] else: - if not args.exchange: - sys.exit("No exchange specified.") config = { 'stake_currency': '', 'dry_run': True, 'exchange': { - 'name': args.exchange, + 'name': exchange_name, 'key': '', 'secret': '', 'pair_whitelist': [], @@ -71,6 +69,13 @@ else: } configuration._load_logging_config(config) + +if args.config and args.exchange: + logger.warning("The --exchange option is ignored, using exchange settings from the configuration file.") + +# Check if the exchange set by the user is supported +configuration.check_exchange(config) + configuration._load_datadir_config(config) dl_path = Path(config['datadir']) From cd60d6d99adc61ca030f418d3a60de9bfbd43fd1 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 11 Jun 2019 10:10:21 +0300 Subject: [PATCH 48/70] make --days positive int only --- freqtrade/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 0d4288ef3..09fea5e63 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -470,7 +470,7 @@ class Arguments(object): '--days', help='Download data for given number of days.', dest='days', - type=int, + type=Arguments.check_int_positive, metavar='INT', ) self.parser.add_argument( From d55f2be942b5f9a58cd085e815c5a11311f802c0 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 11 Jun 2019 10:21:59 +0300 Subject: [PATCH 49/70] make flake happy --- scripts/download_backtest_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 278879fb0..76b2c415b 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -71,7 +71,8 @@ else: configuration._load_logging_config(config) if args.config and args.exchange: - logger.warning("The --exchange option is ignored, using exchange settings from the configuration file.") + logger.warning("The --exchange option is ignored, " + "using exchange settings from the configuration file.") # Check if the exchange set by the user is supported configuration.check_exchange(config) From 4801af4c77afbb10a8ec756e6c22955521b08a17 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 11 Jun 2019 10:42:14 +0300 Subject: [PATCH 50/70] debug logging for IStrategy.advise_*() added --- freqtrade/strategy/interface.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index db266d95f..4e2d96c55 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -378,6 +378,7 @@ class IStrategy(ABC): :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ + logger.debug(f"Populating indicators for pair {metadata.get('pair')}.") if self._populate_fun_len == 2: warnings.warn("deprecated - check out the Sample strategy to see " "the current function headers!", DeprecationWarning) @@ -393,6 +394,7 @@ class IStrategy(ABC): :param pair: Additional information, like the currently traded pair :return: DataFrame with buy column """ + logger.debug(f"Populating buy signals for pair {metadata.get('pair')}.") if self._buy_fun_len == 2: warnings.warn("deprecated - check out the Sample strategy to see " "the current function headers!", DeprecationWarning) @@ -408,6 +410,7 @@ class IStrategy(ABC): :param pair: Additional information, like the currently traded pair :return: DataFrame with sell column """ + logger.debug(f"Populating sell signals for pair {metadata.get('pair')}.") if self._sell_fun_len == 2: warnings.warn("deprecated - check out the Sample strategy to see " "the current function headers!", DeprecationWarning) From 7322a34fa426d0eb42f7ad5711b997ad356c19fa Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 11 Jun 2019 10:58:19 +0300 Subject: [PATCH 51/70] fix metadata in tests --- freqtrade/tests/strategy/test_strategy.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index 2ed2567f9..b96f9c79e 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -63,15 +63,14 @@ def test_search_strategy(): def test_load_strategy(result): resolver = StrategyResolver({'strategy': 'TestStrategy'}) - metadata = {'pair': 'ETH/BTC'} - assert 'adx' in resolver.strategy.advise_indicators(result, metadata=metadata) + assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) def test_load_strategy_byte64(result): with open("freqtrade/tests/strategy/test_strategy.py", "r") as file: encoded_string = urlsafe_b64encode(file.read().encode("utf-8")).decode("utf-8") resolver = StrategyResolver({'strategy': 'TestStrategy:{}'.format(encoded_string)}) - assert 'adx' in resolver.strategy.advise_indicators(result, 'ETH/BTC') + assert 'adx' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) def test_load_strategy_invalid_directory(result, caplog): @@ -371,7 +370,7 @@ def test_deprecate_populate_indicators(result): with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") - indicators = resolver.strategy.advise_indicators(result, 'ETH/BTC') + indicators = resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert "deprecated - check out the Sample strategy to see the current function headers!" \ @@ -380,7 +379,7 @@ def test_deprecate_populate_indicators(result): with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") - resolver.strategy.advise_buy(indicators, 'ETH/BTC') + resolver.strategy.advise_buy(indicators, {'pair': 'ETH/BTC'}) assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert "deprecated - check out the Sample strategy to see the current function headers!" \ @@ -389,7 +388,7 @@ def test_deprecate_populate_indicators(result): with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") - resolver.strategy.advise_sell(indicators, 'ETH_BTC') + resolver.strategy.advise_sell(indicators, {'pair': 'ETH_BTC'}) assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert "deprecated - check out the Sample strategy to see the current function headers!" \ From 676e7300132e15aa55a8cdb844e2f3b582140184 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 11 Jun 2019 13:18:35 +0300 Subject: [PATCH 52/70] enhance check_exchange --- freqtrade/configuration.py | 41 +++++++++++++++++++++------ freqtrade/exchange/__init__.py | 6 ++-- freqtrade/exchange/exchange.py | 18 ++++++++---- freqtrade/tests/test_configuration.py | 2 +- 4 files changed, 50 insertions(+), 17 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index c19580c36..90393cae2 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -13,7 +13,8 @@ from jsonschema import Draft4Validator, validators from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants -from freqtrade.exchange import is_exchange_supported, supported_exchanges +from freqtrade.exchange import (is_exchange_bad, is_exchange_known, + is_exchange_officially_supported, known_exchanges) from freqtrade.misc import deep_merge_dicts from freqtrade.state import RunMode @@ -375,22 +376,44 @@ class Configuration(object): return self.config - def check_exchange(self, config: Dict[str, Any]) -> bool: + def check_exchange(self, config: Dict[str, Any], check_for_bad: bool = True) -> bool: """ Check if the exchange name in the config file is supported by Freqtrade - :return: True or raised an exception if the exchange if not supported + :param check_for_bad: if True, check the exchange against the list of known 'bad' + exchanges + :return: False if exchange is 'bad', i.e. is known to work with the bot with + critical issues or does not work at all, crashes, etc. True otherwise. + raises an exception if the exchange if not supported by ccxt + and thus is not known for the Freqtrade at all. """ - exchange = config.get('exchange', {}).get('name').lower() - if not is_exchange_supported(exchange): + logger.info("Checking exchange...") - exception_msg = f'Exchange "{exchange}" not supported.\n' \ - f'The following exchanges are supported: ' \ - f'{", ".join(supported_exchanges())}' + exchange = config.get('exchange', {}).get('name').lower() + if not is_exchange_known(exchange): + exception_msg = f'Exchange "{exchange}" is not supported by ccxt ' \ + f'and not known for the bot.\n' \ + f'The following exchanges are supported by ccxt: ' \ + f'{", ".join(known_exchanges())}' logger.critical(exception_msg) raise OperationalException( exception_msg ) - logger.debug('Exchange "%s" supported', exchange) + logger.info(f'Exchange "{exchange}" is supported by ccxt and known for the bot.') + + if is_exchange_officially_supported(exchange): + logger.info(f'Exchange "{exchange}" is officially supported ' + f'by the Freqtrade development team.') + else: + logger.warning(f'Exchange "{exchange}" is not officially supported ' + f'by the Freqtrade development team. ' + f'It may work with serious issues or not work at all. ' + f'Use it at your own discretion.') + + if check_for_bad and is_exchange_bad(exchange): + logger.warning(f'Exchange "{exchange}" is known to not work with Freqtrade yet. ' + f'Use it only for development and testing purposes.') + return False + return True diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 3c90e69ee..29e50e29c 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,6 +1,8 @@ from freqtrade.exchange.exchange import Exchange # noqa: F401 -from freqtrade.exchange.exchange import (is_exchange_supported, # noqa: F401 - supported_exchanges) +from freqtrade.exchange.exchange import (is_exchange_bad, # noqa: F401 + is_exchange_known, + is_exchange_officially_supported, + known_exchanges) from freqtrade.exchange.exchange import (timeframe_to_seconds, # noqa: F401 timeframe_to_minutes, timeframe_to_msecs) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ea6996efb..1196f5efe 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -156,8 +156,8 @@ class Exchange(object): # Find matching class for the given exchange name name = exchange_config['name'] - if not is_exchange_supported(name, ccxt_module): - raise OperationalException(f'Exchange {name} is not supported') +# if not is_exchange_supported(name, ccxt_module): +# raise OperationalException(f'Exchange {name} is not supported') ex_config = { 'apiKey': exchange_config.get('key'), @@ -722,11 +722,19 @@ class Exchange(object): raise OperationalException(e) -def is_exchange_supported(exchange: str, ccxt_module=None) -> bool: - return exchange in supported_exchanges(ccxt_module) +def is_exchange_bad(exchange: str) -> bool: + return exchange in ['bitmex'] -def supported_exchanges(ccxt_module=None) -> List[str]: +def is_exchange_known(exchange: str, ccxt_module=None) -> bool: + return exchange in known_exchanges(ccxt_module) + + +def is_exchange_officially_supported(exchange: str) -> bool: + return exchange in ['bittrex', 'binance'] + + +def known_exchanges(ccxt_module=None) -> List[str]: return ccxt_module.exchanges if ccxt_module is not None else ccxt.exchanges diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index aee0dfadd..03f0c004c 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -484,7 +484,7 @@ def test_check_exchange(default_conf, caplog) -> None: with pytest.raises( OperationalException, - match=r'.*Exchange "unknown_exchange" not supported.*' + match=r'.*Exchange "unknown_exchange" is not supported by ccxt and not known for the bot.*' ): configuration.check_exchange(default_conf) From db6ccef6bdad38d3438bc08554ee91d500a514c1 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 11 Jun 2019 13:43:29 +0300 Subject: [PATCH 53/70] return back check in init_ccxt() --- freqtrade/exchange/exchange.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 1196f5efe..0d4ed364d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -156,8 +156,8 @@ class Exchange(object): # Find matching class for the given exchange name name = exchange_config['name'] -# if not is_exchange_supported(name, ccxt_module): -# raise OperationalException(f'Exchange {name} is not supported') + if not is_exchange_known(name, ccxt_module): + raise OperationalException(f'Exchange {name} is not supported by ccxt') ex_config = { 'apiKey': exchange_config.get('key'), From dc7f8837518e280ee950b811c9d2ee8e7826d810 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 11 Jun 2019 13:47:04 +0300 Subject: [PATCH 54/70] no need to duplicate this long error message --- freqtrade/configuration.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 90393cae2..bd4717894 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -390,14 +390,11 @@ class Configuration(object): exchange = config.get('exchange', {}).get('name').lower() if not is_exchange_known(exchange): - exception_msg = f'Exchange "{exchange}" is not supported by ccxt ' \ - f'and not known for the bot.\n' \ - f'The following exchanges are supported by ccxt: ' \ - f'{", ".join(known_exchanges())}' - - logger.critical(exception_msg) raise OperationalException( - exception_msg + f'Exchange "{exchange}" is not supported by ccxt ' + f'and not known for the bot.\n' + f'The following exchanges are supported by ccxt: ' + f'{", ".join(known_exchanges())}' ) logger.info(f'Exchange "{exchange}" is supported by ccxt and known for the bot.') From 0cc2210f222d54127ba58b7fdf7173b32fe0746d Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 12 Jun 2019 22:37:43 +0300 Subject: [PATCH 55/70] wording fixed --- freqtrade/configuration.py | 20 ++++++++++---------- freqtrade/exchange/__init__.py | 4 ++-- freqtrade/exchange/exchange.py | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index bd4717894..b85345b45 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -13,8 +13,8 @@ from jsonschema import Draft4Validator, validators from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants -from freqtrade.exchange import (is_exchange_bad, is_exchange_known, - is_exchange_officially_supported, known_exchanges) +from freqtrade.exchange import (is_exchange_bad, is_exchange_available, + is_exchange_officially_supported, available_exchanges) from freqtrade.misc import deep_merge_dicts from freqtrade.state import RunMode @@ -389,27 +389,27 @@ class Configuration(object): logger.info("Checking exchange...") exchange = config.get('exchange', {}).get('name').lower() - if not is_exchange_known(exchange): + if not is_exchange_available(exchange): raise OperationalException( f'Exchange "{exchange}" is not supported by ccxt ' - f'and not known for the bot.\n' + f'and therefore not available for the bot.\n' f'The following exchanges are supported by ccxt: ' - f'{", ".join(known_exchanges())}' + f'{", ".join(available_exchanges())}' ) - logger.info(f'Exchange "{exchange}" is supported by ccxt and known for the bot.') + logger.info(f'Exchange "{exchange}" is supported by ccxt ' + f'and therefore available for the bot.') if is_exchange_officially_supported(exchange): logger.info(f'Exchange "{exchange}" is officially supported ' f'by the Freqtrade development team.') else: - logger.warning(f'Exchange "{exchange}" is not officially supported ' - f'by the Freqtrade development team. ' - f'It may work with serious issues or not work at all. ' + logger.warning(f'Exchange "{exchange}" is not officially supported. ' + f'It may work flawlessly (please report back) or have serious issues. ' f'Use it at your own discretion.') if check_for_bad and is_exchange_bad(exchange): - logger.warning(f'Exchange "{exchange}" is known to not work with Freqtrade yet. ' + logger.warning(f'Exchange "{exchange}" is known to not work with the bot yet. ' f'Use it only for development and testing purposes.') return False diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 29e50e29c..5c58320f6 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,8 +1,8 @@ from freqtrade.exchange.exchange import Exchange # noqa: F401 from freqtrade.exchange.exchange import (is_exchange_bad, # noqa: F401 - is_exchange_known, + is_exchange_available, is_exchange_officially_supported, - known_exchanges) + available_exchanges) from freqtrade.exchange.exchange import (timeframe_to_seconds, # noqa: F401 timeframe_to_minutes, timeframe_to_msecs) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 0d4ed364d..194e1d883 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -156,7 +156,7 @@ class Exchange(object): # Find matching class for the given exchange name name = exchange_config['name'] - if not is_exchange_known(name, ccxt_module): + if not is_exchange_available(name, ccxt_module): raise OperationalException(f'Exchange {name} is not supported by ccxt') ex_config = { @@ -726,15 +726,15 @@ def is_exchange_bad(exchange: str) -> bool: return exchange in ['bitmex'] -def is_exchange_known(exchange: str, ccxt_module=None) -> bool: - return exchange in known_exchanges(ccxt_module) +def is_exchange_available(exchange: str, ccxt_module=None) -> bool: + return exchange in available_exchanges(ccxt_module) def is_exchange_officially_supported(exchange: str) -> bool: return exchange in ['bittrex', 'binance'] -def known_exchanges(ccxt_module=None) -> List[str]: +def available_exchanges(ccxt_module=None) -> List[str]: return ccxt_module.exchanges if ccxt_module is not None else ccxt.exchanges From a65c89f09074c7c2eba664afe6335f3186788b03 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 12 Jun 2019 23:03:16 +0300 Subject: [PATCH 56/70] test adjusted --- freqtrade/tests/test_configuration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 03f0c004c..c8beb3512 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -484,7 +484,8 @@ def test_check_exchange(default_conf, caplog) -> None: with pytest.raises( OperationalException, - match=r'.*Exchange "unknown_exchange" is not supported by ccxt and not known for the bot.*' + match=r'.*Exchange "unknown_exchange" is not supported by ccxt ' + r'and therefore not available for the bot.*' ): configuration.check_exchange(default_conf) From a4d8424268f0d2ed8fdb03d6cc3f16b51b888215 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Jun 2019 19:34:46 +0200 Subject: [PATCH 57/70] trailing_stop_positive should only be set when needed, and none/undefined otherwise --- user_data/strategies/test_strategy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 66a5f8c09..1dd6406b4 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -44,8 +44,8 @@ class TestStrategy(IStrategy): # trailing stoploss trailing_stop = False - trailing_stop_positive = 0.01 - trailing_stop_positive_offset = 0.0 # Disabled / not configured + # trailing_stop_positive = 0.01 + # trailing_stop_positive_offset = 0.0 # Disabled / not configured # Optimal ticker interval for the strategy ticker_interval = '5m' From b64b6a2583f55f1f1d139a45a59f70bc633c4c61 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Jun 2019 19:35:20 +0200 Subject: [PATCH 58/70] Support trailing_stop_positive options in BTContainer --- freqtrade/tests/optimize/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/tests/optimize/__init__.py b/freqtrade/tests/optimize/__init__.py index 457113cb7..41500051f 100644 --- a/freqtrade/tests/optimize/__init__.py +++ b/freqtrade/tests/optimize/__init__.py @@ -29,6 +29,10 @@ class BTContainer(NamedTuple): trades: List[BTrade] profit_perc: float trailing_stop: bool = False + trailing_only_offset_is_reached: bool = False + trailing_stop_positive: float = None + trailing_stop_positive_offset: float = 0.0 + use_sell_signal: bool = False def _get_frame_time_from_offset(offset): From 578180f45b986aaa5dade8425102aee0baa59dbb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Jun 2019 20:00:56 +0200 Subject: [PATCH 59/70] Add test for sell-signal sell --- .../tests/optimize/test_backtest_detail.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 32c6bd09b..35f705e16 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -14,6 +14,21 @@ from freqtrade.tests.optimize import (BTContainer, BTrade, _get_frame_time_from_offset, tests_ticker_interval) +# Test 0 Sell signal sell +# Test with Stop-loss at 1% +# TC0: Sell signal in candle 3 +tc0 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4986, 4600, 6172, 0, 0], # exit with stoploss hit + [3, 5010, 5000, 4980, 5010, 6172, 0, 1], + [4, 5010, 4987, 4977, 4995, 6172, 0, 0], + [5, 4995, 4995, 4995, 4950, 6172, 0, 0]], + stop_loss=-0.01, roi=1, profit_perc=0.002, use_sell_signal=True, + trades=[BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=1, close_tick=4)] +) + # Test 1 Minus 8% Close # Test with Stop-loss at 1% # TC1: Stop-Loss Triggered 1% loss @@ -146,7 +161,7 @@ tc8 = BTContainer(data=[ # Test 9 - trailing_stop should raise - high and low in same candle. # Candle Data for test 9 # Set stop-loss at 10%, ROI at 10% (should not apply) -# TC9: Trailing stoploss - stoploss should be adjusted candle 2 +# TC9: Trailing stoploss - stoploss should be adjusted candle 3 tc9 = BTContainer(data=[ # D O H L C V B S [0, 5000, 5050, 4950, 5000, 6172, 1, 0], @@ -159,6 +174,7 @@ tc9 = BTContainer(data=[ ) TESTS = [ + tc0, tc1, tc2, tc3, @@ -180,6 +196,13 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: default_conf["minimal_roi"] = {"0": data.roi} default_conf["ticker_interval"] = tests_ticker_interval default_conf["trailing_stop"] = data.trailing_stop + default_conf["trailing_only_offset_is_reached"] = data.trailing_only_offset_is_reached + # Only add this to configuration If it's necessary + if data.trailing_stop_positive: + default_conf["trailing_stop_positive"] = data.trailing_stop_positive + default_conf["trailing_stop_positive_offset"] = data.trailing_stop_positive_offset + default_conf["experimental"] = {"use_sell_signal": data.use_sell_signal} + mocker.patch("freqtrade.exchange.Exchange.get_fee", MagicMock(return_value=0.0)) patch_exchange(mocker) frame = _build_backtest_dataframe(data.data) From 160894c0312cf5c778a3d04b8968f62b28aecd4c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Jun 2019 20:04:52 +0200 Subject: [PATCH 60/70] Calculate profit_high to make sure stoploss_positive_offset is correct --- freqtrade/strategy/interface.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index db266d95f..8570c354f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -308,14 +308,16 @@ class IStrategy(ABC): if trailing_stop: # trailing stoploss handling - sl_offset = self.config.get('trailing_stop_positive_offset') or 0.0 tsl_only_offset = self.config.get('trailing_only_offset_is_reached', False) + # Make sure current_profit is calculated using high for backtesting. + high_profit = current_profit if not high else trade.calc_profit_percent(high) + # Don't update stoploss if trailing_only_offset_is_reached is true. - if not (tsl_only_offset and current_profit < sl_offset): + if not (tsl_only_offset and high_profit < sl_offset): # Specific handling for trailing_stop_positive - if 'trailing_stop_positive' in self.config and current_profit > sl_offset: + if 'trailing_stop_positive' in self.config and high_profit > sl_offset: # Ignore mypy error check in configuration that this is a float stop_loss_value = self.config.get('trailing_stop_positive') # type: ignore logger.debug(f"using positive stop loss: {stop_loss_value} " From 550fbad53e7fc64074744945fea988273c0ebad8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Jun 2019 20:05:49 +0200 Subject: [PATCH 61/70] Add test-cases with trailing_stop_offsets --- .../tests/optimize/test_backtest_detail.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 35f705e16..402e22391 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -173,6 +173,57 @@ tc9 = BTContainer(data=[ trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3)] ) +# Test 10 - trailing_stop should raise so candle 3 causes a stoploss +# without applying trailing_stop_positive since stoploss_offset is at 10%. +# Set stop-loss at 10%, ROI at 10% (should not apply) +# TC10: Trailing stoploss - stoploss should be adjusted candle 2 +tc10 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 4950, 5100, 6172, 0, 0], + [2, 5100, 5251, 5100, 5100, 6172, 0, 0], + [3, 4850, 5050, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi=0.10, profit_perc=-0.1, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.10, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=4)] +) + +# Test 11 - trailing_stop should raise so candle 3 causes a stoploss +# applying a positive trailing stop of 3% since stop_positive_offset is reached. +# Set stop-loss at 10%, ROI at 10% (should not apply) +# TC11: Trailing stoploss - stoploss should be adjusted candle 2, +tc11 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 4950, 5100, 6172, 0, 0], + [2, 5100, 5251, 5100, 5100, 6172, 0, 0], + [3, 4850, 5050, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi=0.10, profit_perc=0.019, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.05, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3)] +) + +# Test 12 - trailing_stop should raise in candle 2 and cause a stoploss in the same candle +# applying a positive trailing stop of 3% since stop_positive_offset is reached. +# Set stop-loss at 10%, ROI at 10% (should not apply) +# TC12: Trailing stoploss - stoploss should be adjusted candle 2, +tc12 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 4950, 5100, 6172, 0, 0], + [2, 5100, 5251, 4650, 5100, 6172, 0, 0], + [3, 4850, 5050, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi=0.10, profit_perc=0.019, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.05, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=2)] +) + TESTS = [ tc0, tc1, @@ -184,6 +235,9 @@ TESTS = [ tc7, tc8, tc9, + tc10, + tc11, + tc12, ] From e08fda074ad2ea504b830c8760ad1a5e198e9b0e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Jun 2019 20:26:47 +0200 Subject: [PATCH 62/70] Fix bug with timeframe handling --- scripts/download_backtest_data.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 76b2c415b..8493c0872 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -30,7 +30,6 @@ args = arguments.parse_args(no_default_config=True) # Use bittrex as default exchange exchange_name = args.exchange or 'bittrex' -timeframes = args.timeframes pairs: List = [] configuration = Configuration(args) @@ -50,7 +49,9 @@ if args.config: config['exchange']['secret'] = '' pairs = config['exchange']['pair_whitelist'] - timeframes = [config['ticker_interval']] + + # Don't fail if ticker_interval is not in the configuration + timeframes = [config.get('ticker_interval')] else: config = { @@ -68,6 +69,8 @@ else: } } +timeframes = args.timeframes + configuration._load_logging_config(config) if args.config and args.exchange: From 9657b1a17f454fcd81f66f91c2c8949c7495acfd Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Jun 2019 20:37:17 +0200 Subject: [PATCH 63/70] explict parse to string for ticker-interval --- scripts/download_backtest_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 8493c0872..a39500d0e 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -127,7 +127,7 @@ try: logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') download_pair_history(datadir=dl_path, exchange=exchange, - pair=pair, ticker_interval=ticker_interval, + pair=pair, ticker_interval=str(ticker_interval), timerange=timerange) except KeyboardInterrupt: From 04ea66c97722901caeae519fd0b38feb886bae1e Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Jun 2019 02:58:34 +0300 Subject: [PATCH 64/70] fix handling timeframes --- freqtrade/arguments.py | 1 - scripts/download_backtest_data.py | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 09fea5e63..46fb9e7b3 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -484,7 +484,6 @@ class Arguments(object): Default: %(default)s.', choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h', '1d', '3d', '1w'], - default=['1m', '5m'], nargs='+', dest='timeframes', ) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index a39500d0e..6263d0e2f 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -50,8 +50,10 @@ if args.config: pairs = config['exchange']['pair_whitelist'] - # Don't fail if ticker_interval is not in the configuration - timeframes = [config.get('ticker_interval')] + if config.get('ticker_interval'): + timeframes = args.timeframes or [config.get('ticker_interval')] + else: + timeframes = args.timeframes or ['1m', '5m'] else: config = { @@ -68,8 +70,7 @@ else: } } } - -timeframes = args.timeframes + timeframes = args.timeframes or ['1m', '5m'] configuration._load_logging_config(config) From ee113ab8edb29269221fd0a10830c1a5339cafde Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Jun 2019 18:40:02 +0300 Subject: [PATCH 65/70] log messages aligned --- freqtrade/configuration.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index b85345b45..28faacf90 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -397,20 +397,19 @@ class Configuration(object): f'{", ".join(available_exchanges())}' ) - logger.info(f'Exchange "{exchange}" is supported by ccxt ' - f'and therefore available for the bot.') - - if is_exchange_officially_supported(exchange): - logger.info(f'Exchange "{exchange}" is officially supported ' - f'by the Freqtrade development team.') - else: - logger.warning(f'Exchange "{exchange}" is not officially supported. ' - f'It may work flawlessly (please report back) or have serious issues. ' - f'Use it at your own discretion.') - if check_for_bad and is_exchange_bad(exchange): logger.warning(f'Exchange "{exchange}" is known to not work with the bot yet. ' f'Use it only for development and testing purposes.') return False + if is_exchange_officially_supported(exchange): + logger.info(f'Exchange "{exchange}" is officially supported ' + f'by the Freqtrade development team.') + else: + logger.warning(f'Exchange "{exchange}" is supported by ccxt ' + f'and therefore available for the bot but not officially supported ' + f'by the Freqtrade development team. ' + f'It may work flawlessly (please report back) or have serious issues. ' + f'Use it at your own discretion.') + return True From 941fb4ebbb3c7f3d53bc3741e20d086b7545b072 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 14 Jun 2019 18:40:25 +0300 Subject: [PATCH 66/70] tests added --- freqtrade/tests/test_configuration.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index c8beb3512..82167125c 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -470,15 +470,27 @@ def test_hyperopt_with_arguments(mocker, default_conf, caplog) -> None: def test_check_exchange(default_conf, caplog) -> None: configuration = Configuration(Namespace()) - # Test a valid exchange + # Test an officially supported by Freqtrade team exchange default_conf.get('exchange').update({'name': 'BITTREX'}) assert configuration.check_exchange(default_conf) - # Test a valid exchange + # Test an officially supported by Freqtrade team exchange default_conf.get('exchange').update({'name': 'binance'}) assert configuration.check_exchange(default_conf) - # Test a invalid exchange + # Test an available exchange, supported by ccxt + default_conf.get('exchange').update({'name': 'kraken'}) + assert configuration.check_exchange(default_conf) + + # Test a 'bad' exchange, which known to have serious problems + default_conf.get('exchange').update({'name': 'bitmex'}) + assert not configuration.check_exchange(default_conf) + + # Test a 'bad' exchange with check_for_bad=False + default_conf.get('exchange').update({'name': 'bitmex'}) + assert configuration.check_exchange(default_conf, False) + + # Test an invalid exchange default_conf.get('exchange').update({'name': 'unknown_exchange'}) configuration.config = default_conf From ad9dc349e46fc0cc6438939f944cedc12aef07cc Mon Sep 17 00:00:00 2001 From: Misagh Date: Sat, 15 Jun 2019 12:20:32 +0200 Subject: [PATCH 67/70] edge cli should override stake_amount --- freqtrade/optimize/edge_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index 8232c79c9..231493e4d 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -6,6 +6,7 @@ This module contains the edge backtesting interface import logging from typing import Dict, Any from tabulate import tabulate +from freqtrade import constants from freqtrade.edge import Edge from freqtrade.arguments import Arguments @@ -32,6 +33,7 @@ class EdgeCli(object): self.config['exchange']['secret'] = '' self.config['exchange']['password'] = '' self.config['exchange']['uid'] = '' + self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT self.config['dry_run'] = True self.exchange = Exchange(self.config) self.strategy = StrategyResolver(self.config).strategy From 707118a63684290287fbfc53c6dcfd60e7e7f5e3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Jun 2019 13:04:15 +0200 Subject: [PATCH 68/70] Test stake changed to unlimited --- freqtrade/tests/optimize/test_edge_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/tests/optimize/test_edge_cli.py b/freqtrade/tests/optimize/test_edge_cli.py index 5d16b0f2d..49d3cdd55 100644 --- a/freqtrade/tests/optimize/test_edge_cli.py +++ b/freqtrade/tests/optimize/test_edge_cli.py @@ -117,8 +117,10 @@ def test_start(mocker, fee, edge_conf, caplog) -> None: def test_edge_init(mocker, edge_conf) -> None: patch_exchange(mocker) + edge_conf['stake_amount'] = 20 edge_cli = EdgeCli(edge_conf) assert edge_cli.config == edge_conf + assert edge_cli.config['stake_amount'] == 'unlimited' assert callable(edge_cli.edge.calculate) From a77d75eb431c29123b537b7dc1056bca1809560c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Jun 2019 13:14:07 +0200 Subject: [PATCH 69/70] Check log output since that's whats shown to users --- freqtrade/tests/test_configuration.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 82167125c..38f17fbea 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -15,7 +15,7 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration, set_loggers from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.state import RunMode -from freqtrade.tests.conftest import log_has +from freqtrade.tests.conftest import log_has, log_has_re @pytest.fixture(scope="function") @@ -473,22 +473,40 @@ def test_check_exchange(default_conf, caplog) -> None: # Test an officially supported by Freqtrade team exchange default_conf.get('exchange').update({'name': 'BITTREX'}) assert configuration.check_exchange(default_conf) + assert log_has_re(r"Exchange .* is officially supported by the Freqtrade development team\.", + caplog.record_tuples) + caplog.clear() # Test an officially supported by Freqtrade team exchange default_conf.get('exchange').update({'name': 'binance'}) assert configuration.check_exchange(default_conf) + assert log_has_re(r"Exchange .* is officially supported by the Freqtrade development team\.", + caplog.record_tuples) + caplog.clear() # Test an available exchange, supported by ccxt default_conf.get('exchange').update({'name': 'kraken'}) assert configuration.check_exchange(default_conf) + assert log_has_re(r"Exchange .* is supported by ccxt and .* not officially supported " + r"by the Freqtrade development team\. .*", + caplog.record_tuples) + caplog.clear() # Test a 'bad' exchange, which known to have serious problems default_conf.get('exchange').update({'name': 'bitmex'}) assert not configuration.check_exchange(default_conf) + assert log_has_re(r"Exchange .* is known to not work with the bot yet\. " + r"Use it only for development and testing purposes\.", + caplog.record_tuples) + caplog.clear() # Test a 'bad' exchange with check_for_bad=False default_conf.get('exchange').update({'name': 'bitmex'}) assert configuration.check_exchange(default_conf, False) + assert log_has_re(r"Exchange .* is supported by ccxt and .* not officially supported " + r"by the Freqtrade development team\. .*", + caplog.record_tuples) + caplog.clear() # Test an invalid exchange default_conf.get('exchange').update({'name': 'unknown_exchange'}) From 36dd061be759167f1b376fbf94ec6bd93e77cfd7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Jun 2019 13:19:18 +0200 Subject: [PATCH 70/70] Update slack link since the old one expired --- CONTRIBUTING.md | 2 +- README.md | 4 ++-- docs/developer.md | 2 +- docs/index.md | 2 +- docs/strategy-customization.md | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3c511f44d..e15059f56 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ Few pointers for contributions: - Create your PR against the `develop` branch, not `master`. - New features need to contain unit tests and must be PEP8 conformant (max-line-length = 100). -If you are unsure, discuss the feature on our [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) +If you are unsure, discuss the feature on our [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LWEyODBiNzkzNzcyNzU0MWYyYzE5NjIyOTQxMzBmMGUxOTIzM2YyN2Y4NWY1YTEwZDgwYTRmMzE2NmM5ZmY2MTg) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. ## Getting started diff --git a/README.md b/README.md index 98dad1d2e..240b4f917 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ Accounts having BNB accounts use this to pay for fees - if your first trade happ For any questions not covered by the documentation or for further information about the bot, we encourage you to join our slack channel. -- [Click here to join Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE). +- [Click here to join Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LWEyODBiNzkzNzcyNzU0MWYyYzE5NjIyOTQxMzBmMGUxOTIzM2YyN2Y4NWY1YTEwZDgwYTRmMzE2NmM5ZmY2MTg). ### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) @@ -172,7 +172,7 @@ to understand the requirements before sending your pull-requests. Coding is not a neccessity to contribute - maybe start with improving our documentation? Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase. -**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. +**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LWEyODBiNzkzNzcyNzU0MWYyYzE5NjIyOTQxMzBmMGUxOTIzM2YyN2Y4NWY1YTEwZDgwYTRmMzE2NmM5ZmY2MTg). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. **Important:** Always create your PR against the `develop` branch, not `master`. diff --git a/docs/developer.md b/docs/developer.md index 6ecb7f156..7f3dc76f6 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -2,7 +2,7 @@ This page is intended for developers of FreqTrade, people who want to contribute to the FreqTrade codebase or documentation, or people who want to understand the source code of the application they're running. -All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel in [slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) where you can ask questions. +All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel in [slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LWEyODBiNzkzNzcyNzU0MWYyYzE5NjIyOTQxMzBmMGUxOTIzM2YyN2Y4NWY1YTEwZDgwYTRmMzE2NmM5ZmY2MTg) where you can ask questions. ## Documentation diff --git a/docs/index.md b/docs/index.md index 9fbc0519c..63d6be75e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -64,7 +64,7 @@ To run this bot we recommend you a cloud instance with a minimum of: Help / Slack For any questions not covered by the documentation or for further information about the bot, we encourage you to join our Slack channel. -Click [here](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) to join Slack channel. +Click [here](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LWEyODBiNzkzNzcyNzU0MWYyYzE5NjIyOTQxMzBmMGUxOTIzM2YyN2Y4NWY1YTEwZDgwYTRmMzE2NmM5ZmY2MTg) to join Slack channel. ## Ready to try? diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index fd9760bda..57c646aed 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -410,7 +410,7 @@ To get additional Ideas for strategies, head over to our [strategy repository](h Feel free to use any of them as inspiration for your own strategies. We're happy to accept Pull Requests containing new Strategies to that repo. -We also got a *strategy-sharing* channel in our [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) which is a great place to get and/or share ideas. +We also got a *strategy-sharing* channel in our [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LWEyODBiNzkzNzcyNzU0MWYyYzE5NjIyOTQxMzBmMGUxOTIzM2YyN2Y4NWY1YTEwZDgwYTRmMzE2NmM5ZmY2MTg) which is a great place to get and/or share ideas. ## Next step