From 768a24c375014f755fa857c4b09c8bf4e0154922 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Dec 2020 07:45:41 +0100 Subject: [PATCH 001/183] Add stoplossvalue interface --- docs/strategy-advanced.md | 5 +++++ freqtrade/strategy/interface.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 359280694..85a5a6bc6 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -8,6 +8,11 @@ If you're just getting started, please be familiar with the methods described in !!! Note All callback methods described below should only be implemented in a strategy if they are actually used. +## Custom stoploss logic + +// TODO: Complete this section + + ## Custom order timeout rules Simple, timebased order-timeouts can be configured either via strategy or in the configuration in the `unfilledtimeout` section. diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 027c5d36e..33a7ef0c1 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -254,6 +254,24 @@ class IStrategy(ABC): """ return True + def stoploss_value(self, pair: str, trade: Trade, rate: float, **kwargs) -> float: + """ + Define custom stoploss logic + The custom stoploss can never be below self.stoploss, which serves as a hard maximum loss. + + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, returns the initial stoploss value + + :param pair: Pair that's about to be sold. + :param trade: trade object. + :param rate: Rate that's going to be used when using limit orders + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return float: New stoploss value, relative to the open price + """ + return self.stoploss + def informative_pairs(self) -> ListPairsWithTimeframes: """ Define additional, informative pair/interval combinations to be cached from the exchange. From 8f6aefb591f4d69e8a023c935e3994aab5e3b60c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 11 Dec 2020 07:41:06 +0100 Subject: [PATCH 002/183] Extract stoploss assignment --- freqtrade/persistence/models.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 7fa894e9c..e803b4383 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -342,6 +342,12 @@ class Trade(_DECL_BASE): self.max_rate = max(current_price, self.max_rate or self.open_rate) self.min_rate = min(current_price, self.min_rate or self.open_rate) + def _set_new_stoploss(self, new_loss: float, stoploss: float): + """Assign new stop value""" + self.stop_loss = new_loss + self.stop_loss_pct = -1 * abs(stoploss) + self.stoploss_last_update = datetime.utcnow() + def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False) -> None: """ @@ -360,19 +366,15 @@ class Trade(_DECL_BASE): # no stop loss assigned yet if not self.stop_loss: logger.debug(f"{self.pair} - Assigning new stoploss...") - self.stop_loss = new_loss - self.stop_loss_pct = -1 * abs(stoploss) + self._set_new_stoploss(new_loss, stoploss) self.initial_stop_loss = new_loss self.initial_stop_loss_pct = -1 * abs(stoploss) - self.stoploss_last_update = datetime.utcnow() # evaluate if the stop loss needs to be updated else: if new_loss > self.stop_loss: # stop losses only walk up, never down! logger.debug(f"{self.pair} - Adjusting stoploss...") - self.stop_loss = new_loss - self.stop_loss_pct = -1 * abs(stoploss) - self.stoploss_last_update = datetime.utcnow() + self._set_new_stoploss(new_loss, stoploss) else: logger.debug(f"{self.pair} - Keeping current stoploss...") From a414b57d5469402f4002d7affbf69fedb61b9f11 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 12 Dec 2020 07:06:16 +0100 Subject: [PATCH 003/183] Experiment with custom stoploss interface --- freqtrade/strategy/interface.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 33a7ef0c1..7f60ba62f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -89,6 +89,7 @@ class IStrategy(ABC): trailing_stop_positive: Optional[float] = None trailing_stop_positive_offset: float = 0.0 trailing_only_offset_is_reached = False + custom_stoploss: bool = False # associated timeframe ticker_interval: str # DEPRECATED @@ -254,21 +255,22 @@ class IStrategy(ABC): """ return True - def stoploss_value(self, pair: str, trade: Trade, rate: float, **kwargs) -> float: + def stoploss_value(self, pair: str, trade: Trade, current_rate: float, current_profit: float, + **kwargs) -> float: """ Define custom stoploss logic The custom stoploss can never be below self.stoploss, which serves as a hard maximum loss. - For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ When not implemented by a strategy, returns the initial stoploss value :param pair: Pair that's about to be sold. :param trade: trade object. - :param rate: Rate that's going to be used when using limit orders + :param current_rate: Rate, calculated based on pricing settings in ask_strategy. + :param current_profit: Current profit (as ratio), calculated based on current_rate. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - :return float: New stoploss value, relative to the open price + :return float: New stoploss value, relative to the currentrate """ return self.stoploss @@ -549,6 +551,18 @@ class IStrategy(ABC): # Initiate stoploss with open_rate. Does nothing if stoploss is already set. trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True) + if self.custom_stoploss: + stop_loss_value = strategy_safe_wrapper(self.stoploss_value, default_retval=None + )(pair=trade.pair, trade=trade, + current_rate=current_rate, + current_profit=current_profit) + # Sanity check - error cases will return None + if stop_loss_value: + # logger.info(f"{trade.pair} {stop_loss_value=} {current_profit=}") + trade.adjust_stop_loss(current_rate, stop_loss_value) + else: + logger.warning("CustomStoploss function did not return valid stoploss") + if self.trailing_stop: # trailing stoploss handling sl_offset = self.trailing_stop_positive_offset From 18795844d818e983cdda0041799659dad7a7f963 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 18 Dec 2020 07:59:15 +0100 Subject: [PATCH 004/183] Add initial set of custom stoploss documentation --- docs/strategy-advanced.md | 48 +++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 85a5a6bc6..d4068c82b 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -8,16 +8,56 @@ If you're just getting started, please be familiar with the methods described in !!! Note All callback methods described below should only be implemented in a strategy if they are actually used. -## Custom stoploss logic +## Custom stoploss -// TODO: Complete this section +A stoploss can only ever move upwards - so if you set it to an absolute profit of 2%, you can never move it below this price. +Also, the traditional `stoploss` value serves as an absolute lower level and will be instated as the initial stoploss. +The usage of the custom stoploss method must be enabled by setting `custom_stoploss=True` on the strategy object. +The method must return a stoploss value (float / number) with a relative ratio below the current price. +E.g. `current_profit = 0.05` (5% profit) - stoploss returns `0.02` - then you "locked in" a profit of 3% (`0.05 - 0.02 = 0.03`). + +``` python + custom_stoploss = True + + def stoploss_value(self, pair: str, trade: Trade, current_rate: float, current_profit: float, + **kwargs) -> float: + # TODO: Add full docstring here + return 0.04 +``` + +!!! Tip "Trailing stoploss" + It's recommended to disable `trailing_stop` when using custom stoploss values. Both can work in tandem, but you might encounter the trailing stop to move the price higher while your custom function would not want this, causing conflicting behavior. + +### Custom stoploss examples + +Absolute stoploss. The below example sets absolute profit levels based on the current profit. + +* Use the regular stoploss until 20% profit is reached +* Once profit is > 20% - stoploss will be set to 7%.s +* Once profit is > 25% - stoploss will be 15%. +* Once profit is > 40%, stoploss will be at 25%, locking in at least 25% of the profit. + +``` python + def stoploss_value(self, pair: str, trade: Trade, current_rate: float, current_profit: float, + **kwargs) -> float: + # TODO: Add full docstring here + + # Calculate as `-desired_stop_from_open + current_profit` to get the distance between current_profit and initial price + if current_profit > 0.20: + return (-0.7 + current_profit) + if current_profit > 0.25: + return (-0.15 + current_profit) + if current_profit > 0.40: + return (-0.25 + current_profit) + return 1 +``` ## Custom order timeout rules -Simple, timebased order-timeouts can be configured either via strategy or in the configuration in the `unfilledtimeout` section. +Simple, time-based order-timeouts can be configured either via strategy or in the configuration in the `unfilledtimeout` section. -However, freqtrade also offers a custom callback for both ordertypes, which allows you to decide based on custom criteria if a order did time out or not. +However, freqtrade also offers a custom callback for both order types, which allows you to decide based on custom criteria if a order did time out or not. !!! Note Unfilled order timeouts are not relevant during backtesting or hyperopt, and are only relevant during real (live) trading. Therefore these methods are only called in these circumstances. From f235ab8cf4fb148c122b20822f135cb5ce0a8c2a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Dec 2020 11:39:21 +0100 Subject: [PATCH 005/183] Fix some typos in docs --- docs/strategy-advanced.md | 12 ++++++------ docs/strategy-customization.md | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index d4068c82b..02ee9b201 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -73,7 +73,7 @@ The function must return either `True` (cancel order) or `False` (keep order ali from datetime import datetime, timedelta from freqtrade.persistence import Trade -class Awesomestrategy(IStrategy): +class AwesomeStrategy(IStrategy): # ... populate_* methods @@ -112,7 +112,7 @@ class Awesomestrategy(IStrategy): from datetime import datetime from freqtrade.persistence import Trade -class Awesomestrategy(IStrategy): +class AwesomeStrategy(IStrategy): # ... populate_* methods @@ -148,7 +148,7 @@ This can be used to perform calculations which are pair independent (apply to al ``` python import requests -class Awesomestrategy(IStrategy): +class AwesomeStrategy(IStrategy): # ... populate_* methods @@ -173,7 +173,7 @@ class Awesomestrategy(IStrategy): `confirm_trade_entry()` can be used to abort a trade entry at the latest second (maybe because the price is not what we expect). ``` python -class Awesomestrategy(IStrategy): +class AwesomeStrategy(IStrategy): # ... populate_* methods @@ -209,7 +209,7 @@ class Awesomestrategy(IStrategy): from freqtrade.persistence import Trade -class Awesomestrategy(IStrategy): +class AwesomeStrategy(IStrategy): # ... populate_* methods @@ -264,4 +264,4 @@ class MyAwesomeStrategy2(MyAwesomeStrategy): trailing_stop = True ``` -Both attributes and methods may be overriden, altering behavior of the original strategy in a way you need. +Both attributes and methods may be overridden, altering behavior of the original strategy in a way you need. diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index ab64d3a67..688a1c338 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -309,7 +309,7 @@ Storing information can be accomplished by creating a new dictionary within the The name of the variable can be chosen at will, but should be prefixed with `cust_` to avoid naming collisions with predefined strategy variables. ```python -class Awesomestrategy(IStrategy): +class AwesomeStrategy(IStrategy): # Create custom dictionary cust_info = {} From f7b54c24158f4e80ed927d4a333c739f97e09ce3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Dec 2020 11:46:49 +0100 Subject: [PATCH 006/183] Allow and document time-based custom stoploss closes #3206 --- docs/strategy-advanced.md | 28 +++++++++++++++++++++++++--- freqtrade/strategy/interface.py | 5 +++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 02ee9b201..60da11207 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -31,7 +31,9 @@ E.g. `current_profit = 0.05` (5% profit) - stoploss returns `0.02` - then you "l ### Custom stoploss examples -Absolute stoploss. The below example sets absolute profit levels based on the current profit. +#### Absolute stoploss + +The below example sets absolute profit levels based on the current profit. * Use the regular stoploss until 20% profit is reached * Once profit is > 20% - stoploss will be set to 7%.s @@ -39,8 +41,10 @@ Absolute stoploss. The below example sets absolute profit levels based on the cu * Once profit is > 40%, stoploss will be at 25%, locking in at least 25% of the profit. ``` python - def stoploss_value(self, pair: str, trade: Trade, current_rate: float, current_profit: float, - **kwargs) -> float: + custom_stoploss = True + + def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> float: # TODO: Add full docstring here # Calculate as `-desired_stop_from_open + current_profit` to get the distance between current_profit and initial price @@ -53,6 +57,24 @@ Absolute stoploss. The below example sets absolute profit levels based on the cu return 1 ``` +#### Time based trailing stop + +Use the initial stoploss for the first 60 minutes, after this change to 10% trailing stoploss, and after 2 hours (120 minutes) we use a 5% trailing stoploss. + +``` python + custom_stoploss = True + + def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> float: + # TODO: Add full docstring here + + if current_time - timedelta(minutes=60) > trade.open_time: + return -0.10 + elif current_time - timedelta(minutes=120) > trade.open_time: + return -0.05 + return 1 +``` + ## Custom order timeout rules Simple, time-based order-timeouts can be configured either via strategy or in the configuration in the `unfilledtimeout` section. diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 7f60ba62f..4574ca9f2 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -255,8 +255,8 @@ class IStrategy(ABC): """ return True - def stoploss_value(self, pair: str, trade: Trade, current_rate: float, current_profit: float, - **kwargs) -> float: + def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> float: """ Define custom stoploss logic The custom stoploss can never be below self.stoploss, which serves as a hard maximum loss. @@ -554,6 +554,7 @@ class IStrategy(ABC): if self.custom_stoploss: stop_loss_value = strategy_safe_wrapper(self.stoploss_value, default_retval=None )(pair=trade.pair, trade=trade, + current_time=current_time, current_rate=current_rate, current_profit=current_profit) # Sanity check - error cases will return None From b2c109831668936cae9ea1c3eae54ffc763efd2d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Dec 2020 11:58:42 +0100 Subject: [PATCH 007/183] more docs for dynamic stoploss method --- docs/strategy-advanced.md | 41 +++++++++++++++++++++++++++++---- freqtrade/strategy/interface.py | 1 + 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 60da11207..f54bec206 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -20,23 +20,29 @@ E.g. `current_profit = 0.05` (5% profit) - stoploss returns `0.02` - then you "l ``` python custom_stoploss = True - def stoploss_value(self, pair: str, trade: Trade, current_rate: float, current_profit: float, - **kwargs) -> float: + def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> float: # TODO: Add full docstring here return 0.04 ``` +!!! Note "Use of dates" + All time-based calculations should be done based on `current_time` - using `datetime.now()` or `datetime.utcnow()` is discouraged, as this will break backtesting support. + !!! Tip "Trailing stoploss" It's recommended to disable `trailing_stop` when using custom stoploss values. Both can work in tandem, but you might encounter the trailing stop to move the price higher while your custom function would not want this, causing conflicting behavior. ### Custom stoploss examples +The next section will show some examples on what's possible with the custom stoploss function. +Of course, many more things are possible, and all examples can be combined at will. + #### Absolute stoploss The below example sets absolute profit levels based on the current profit. * Use the regular stoploss until 20% profit is reached -* Once profit is > 20% - stoploss will be set to 7%.s +* Once profit is > 20% - stoploss will be set to 7%. * Once profit is > 25% - stoploss will be 15%. * Once profit is > 40%, stoploss will be at 25%, locking in at least 25% of the profit. @@ -68,13 +74,34 @@ Use the initial stoploss for the first 60 minutes, after this change to 10% trai current_profit: float, **kwargs) -> float: # TODO: Add full docstring here - if current_time - timedelta(minutes=60) > trade.open_time: + if current_time - timedelta(minutes=60) > trade.open_date: return -0.10 - elif current_time - timedelta(minutes=120) > trade.open_time: + elif current_time - timedelta(minutes=120) > trade.open_date: return -0.05 return 1 ``` +#### Different stoploss per pair + +Use a different stoploss depending on the pair. +In this example, we'll trail the highest price with 10% trailing stoploss for `ETH/BTC` and `XRP/BTC`, with 5% trailing stoploss for `LTC/BTC` and with 15% for all other pairs. + +``` python + custom_stoploss = True + + def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> float: + # TODO: Add full docstring here + + if pair in ('ETH/BTC', 'XRP/BTC'): + return -0.10 + elif pair in ('LTC/BTC'): + return -0.05 + return -0.15 +``` + +--- + ## Custom order timeout rules Simple, time-based order-timeouts can be configured either via strategy or in the configuration in the `unfilledtimeout` section. @@ -162,6 +189,8 @@ class AwesomeStrategy(IStrategy): return False ``` +--- + ## Bot loop start callback A simple callback which is called once at the start of every bot throttling iteration. @@ -267,6 +296,8 @@ class AwesomeStrategy(IStrategy): ``` +--- + ## Derived strategies The strategies can be derived from other strategies. This avoids duplication of your custom strategy code. You can use this technique to override small parts of your main strategy, leaving the rest untouched: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4574ca9f2..021674bb9 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -267,6 +267,7 @@ class IStrategy(ABC): :param pair: Pair that's about to be sold. :param trade: trade object. + :param current_time: datetime object, containing the current datetime :param current_rate: Rate, calculated based on pricing settings in ask_strategy. :param current_profit: Current profit (as ratio), calculated based on current_rate. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. From 6892c08e9b9091ab68a89ea92ff654316813b7a0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Dec 2020 13:18:06 +0100 Subject: [PATCH 008/183] Improve docstring --- freqtrade/strategy/interface.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 021674bb9..9378f1996 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -258,12 +258,14 @@ class IStrategy(ABC): def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs) -> float: """ - Define custom stoploss logic + Custom stoploss logic, returning the new distance relative to current_rate (as ratio). + e.g. returning -0.05 would create a stoploss 5% below current_rate. The custom stoploss can never be below self.stoploss, which serves as a hard maximum loss. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ When not implemented by a strategy, returns the initial stoploss value + Only called when custom_stoploss is set to True. :param pair: Pair that's about to be sold. :param trade: trade object. From 11e29156210828e1d3d97714eb45cfc2fcc237cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Dec 2020 17:40:01 +0100 Subject: [PATCH 009/183] Fix documentation problem --- docs/strategy-advanced.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index f54bec206..79a1f0ec5 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -42,9 +42,9 @@ Of course, many more things are possible, and all examples can be combined at wi The below example sets absolute profit levels based on the current profit. * Use the regular stoploss until 20% profit is reached -* Once profit is > 20% - stoploss will be set to 7%. -* Once profit is > 25% - stoploss will be 15%. * Once profit is > 40%, stoploss will be at 25%, locking in at least 25% of the profit. +* Once profit is > 25% - stoploss will be 15%. +* Once profit is > 20% - stoploss will be set to 7%. ``` python custom_stoploss = True @@ -54,12 +54,12 @@ The below example sets absolute profit levels based on the current profit. # TODO: Add full docstring here # Calculate as `-desired_stop_from_open + current_profit` to get the distance between current_profit and initial price - if current_profit > 0.20: - return (-0.7 + current_profit) - if current_profit > 0.25: - return (-0.15 + current_profit) if current_profit > 0.40: return (-0.25 + current_profit) + if current_profit > 0.25: + return (-0.15 + current_profit) + if current_profit > 0.20: + return (-0.7 + current_profit) return 1 ``` @@ -74,10 +74,11 @@ Use the initial stoploss for the first 60 minutes, after this change to 10% trai current_profit: float, **kwargs) -> float: # TODO: Add full docstring here - if current_time - timedelta(minutes=60) > trade.open_date: - return -0.10 - elif current_time - timedelta(minutes=120) > trade.open_date: + # Make sure you have the longest interval first - these conditions are evaluated from top to bottom. + if current_time - timedelta(minutes=120) > trade.open_date: return -0.05 + elif current_time - timedelta(minutes=60) > trade.open_date: + return -0.10 return 1 ``` From ea4238e86019a275f2f8131cd81d6e0cb51ebac8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Dec 2020 17:59:49 +0100 Subject: [PATCH 010/183] cleanup some tests --- tests/strategy/test_default_strategy.py | 19 ++++++++++++++++++- tests/strategy/test_interface.py | 6 ++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py index 1b1648db9..c5d76b4c5 100644 --- a/tests/strategy/test_default_strategy.py +++ b/tests/strategy/test_default_strategy.py @@ -1,3 +1,5 @@ +from datetime import datetime +from freqtrade.persistence.models import Trade from pandas import DataFrame from .strats.default_strategy import DefaultStrategy @@ -12,7 +14,7 @@ def test_default_strategy_structure(): assert hasattr(DefaultStrategy, 'populate_sell_trend') -def test_default_strategy(result): +def test_default_strategy(result, fee): strategy = DefaultStrategy({}) metadata = {'pair': 'ETH/BTC'} @@ -23,3 +25,18 @@ def test_default_strategy(result): assert type(indicators) is DataFrame assert type(strategy.populate_buy_trend(indicators, metadata)) is DataFrame assert type(strategy.populate_sell_trend(indicators, metadata)) is DataFrame + + trade = Trade( + open_rate=19_000, + amount=0.1, + pair='ETH/BTC', + fee_open=fee.return_value + ) + + assert strategy.confirm_trade_entry(pair='ETH/BTC', order_type='limit', amount=0.1, + rate=20000, time_in_force='gtc') is True + assert strategy.confirm_trade_exit(pair='ETH/BTC', trade=trade, order_type='limit', amount=0.1, + rate=20000, time_in_force='gtc', sell_reason='roi') is True + + assert strategy.stoploss_value(pair='ETH/BTC', trade=trade, current_time=datetime.now(), + current_rate=20_000, current_profit=0.05) == strategy.stoploss diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 640849ba4..3625c964f 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -105,9 +105,7 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history): assert log_has('Outdated history for pair xyz. Last tick is 16 minutes old', caplog) -def test_assert_df_raise(default_conf, mocker, caplog, ohlcv_history): - # default_conf defines a 5m interval. we check interval * 2 + 5m - # this is necessary as the last candle is removed (partial candles) by default +def test_assert_df_raise(mocker, caplog, ohlcv_history): ohlcv_history.loc[1, 'date'] = arrow.utcnow().shift(minutes=-16) # Take a copy to correctly modify the call mocked_history = ohlcv_history.copy() @@ -127,7 +125,7 @@ def test_assert_df_raise(default_conf, mocker, caplog, ohlcv_history): caplog) -def test_assert_df(default_conf, mocker, ohlcv_history, caplog): +def test_assert_df(ohlcv_history, caplog): df_len = len(ohlcv_history) - 1 # Ensure it's running when passed correctly _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history), From 22d64553c91e3f3d6c5c2541ee3363c611a8c6f9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Dec 2020 18:00:44 +0100 Subject: [PATCH 011/183] Rename test file --- tests/strategy/{test_strategy.py => test_strategy_loading.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/strategy/{test_strategy.py => test_strategy_loading.py} (100%) diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy_loading.py similarity index 100% rename from tests/strategy/test_strategy.py rename to tests/strategy/test_strategy_loading.py From 5f8610b28f9b3367ac371a1a405490d27a16265d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Dec 2020 20:08:03 +0100 Subject: [PATCH 012/183] Add explicit test for stop_loss_reached --- tests/strategy/test_interface.py | 57 ++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 3625c964f..4aedc0c3a 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -1,4 +1,5 @@ # pragma pylint: disable=missing-docstring, C0103 +from freqtrade.strategy.interface import SellCheckTuple, SellType import logging from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock @@ -286,6 +287,62 @@ def test_min_roi_reached3(default_conf, fee) -> None: assert strategy.min_roi_reached(trade, 0.31, arrow.utcnow().shift(minutes=-2).datetime) +@pytest.mark.parametrize('profit,adjusted,expected,trailing,custom,profit2,adjusted2,expected2', [ + # Profit, adjusted stoploss(absolute), profit for 2nd call, enable trailing, + # enable custom stoploss, expected after 1st call, expected after 2nd call + (0.2, 0.9, SellType.NONE, False, False, 0.3, 0.9, SellType.NONE), + (0.2, 0.9, SellType.NONE, False, False, -0.2, 0.9, SellType.STOP_LOSS), + (0.2, 1.14, SellType.NONE, True, False, 0.05, 1.14, SellType.TRAILING_STOP_LOSS), + (0.01, 0.96, SellType.NONE, True, False, 0.05, 1, SellType.NONE), + (0.05, 1, SellType.NONE, True, False, -0.01, 1, SellType.TRAILING_STOP_LOSS), + # Default custom case - trails with 10% + (0.05, 0.95, SellType.NONE, False, True, -0.02, 0.95, SellType.NONE), + (0.05, 0.95, SellType.NONE, False, True, -0.06, 0.95, SellType.TRAILING_STOP_LOSS), +]) +def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, trailing, custom, + profit2, adjusted2, expected2) -> None: + + default_conf.update({'strategy': 'DefaultStrategy'}) + + strategy = StrategyResolver.load_strategy(default_conf) + trade = Trade( + pair='ETH/BTC', + stake_amount=0.01, + amount=1, + open_date=arrow.utcnow().shift(hours=-1).datetime, + fee_open=fee.return_value, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + ) + trade.adjust_min_max_rates(trade.open_rate) + strategy.trailing_stop = trailing + strategy.trailing_stop_positive = -0.05 + strategy.custom_stoploss = custom + + now = arrow.utcnow().datetime + sl_flag = strategy.stop_loss_reached(current_rate=trade.open_rate * (1 + profit), trade=trade, + current_time=now, current_profit=profit, + force_stoploss=0, high=None) + assert isinstance(sl_flag, SellCheckTuple) + assert sl_flag.sell_type == expected + if expected == SellType.NONE: + assert sl_flag.sell_flag is False + else: + assert sl_flag.sell_flag is True + assert round(trade.stop_loss, 2) == adjusted + + sl_flag = strategy.stop_loss_reached(current_rate=trade.open_rate * (1 + profit2), trade=trade, + current_time=now, current_profit=profit2, + force_stoploss=0, high=None) + assert sl_flag.sell_type == expected2 + if expected2 == SellType.NONE: + assert sl_flag.sell_flag is False + else: + assert sl_flag.sell_flag is True + assert round(trade.stop_loss, 2) == adjusted2 + + def test_analyze_ticker_default(ohlcv_history, mocker, caplog) -> None: caplog.set_level(logging.DEBUG) ind_mock = MagicMock(side_effect=lambda x, meta: x) From f8639fe93849d9423d1ee8e04c90f9b8738334a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 19 Dec 2020 20:22:32 +0100 Subject: [PATCH 013/183] Add more tests for custom_loss --- tests/strategy/test_interface.py | 44 ++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 4aedc0c3a..e8a3ee8de 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -11,7 +11,7 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import load_data -from freqtrade.exceptions import StrategyError +from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.persistence import PairLocks, Trade from freqtrade.resolvers import StrategyResolver from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper @@ -287,20 +287,30 @@ def test_min_roi_reached3(default_conf, fee) -> None: assert strategy.min_roi_reached(trade, 0.31, arrow.utcnow().shift(minutes=-2).datetime) -@pytest.mark.parametrize('profit,adjusted,expected,trailing,custom,profit2,adjusted2,expected2', [ - # Profit, adjusted stoploss(absolute), profit for 2nd call, enable trailing, - # enable custom stoploss, expected after 1st call, expected after 2nd call - (0.2, 0.9, SellType.NONE, False, False, 0.3, 0.9, SellType.NONE), - (0.2, 0.9, SellType.NONE, False, False, -0.2, 0.9, SellType.STOP_LOSS), - (0.2, 1.14, SellType.NONE, True, False, 0.05, 1.14, SellType.TRAILING_STOP_LOSS), - (0.01, 0.96, SellType.NONE, True, False, 0.05, 1, SellType.NONE), - (0.05, 1, SellType.NONE, True, False, -0.01, 1, SellType.TRAILING_STOP_LOSS), - # Default custom case - trails with 10% - (0.05, 0.95, SellType.NONE, False, True, -0.02, 0.95, SellType.NONE), - (0.05, 0.95, SellType.NONE, False, True, -0.06, 0.95, SellType.TRAILING_STOP_LOSS), -]) +@pytest.mark.parametrize( + 'profit,adjusted,expected,trailing,custom,profit2,adjusted2,expected2,custom_stop', [ + # Profit, adjusted stoploss(absolute), profit for 2nd call, enable trailing, + # enable custom stoploss, expected after 1st call, expected after 2nd call + (0.2, 0.9, SellType.NONE, False, False, 0.3, 0.9, SellType.NONE, None), + (0.2, 0.9, SellType.NONE, False, False, -0.2, 0.9, SellType.STOP_LOSS, None), + (0.2, 1.14, SellType.NONE, True, False, 0.05, 1.14, SellType.TRAILING_STOP_LOSS, None), + (0.01, 0.96, SellType.NONE, True, False, 0.05, 1, SellType.NONE, None), + (0.05, 1, SellType.NONE, True, False, -0.01, 1, SellType.TRAILING_STOP_LOSS, None), + # Default custom case - trails with 10% + (0.05, 0.95, SellType.NONE, False, True, -0.02, 0.95, SellType.NONE, None), + (0.05, 0.95, SellType.NONE, False, True, -0.06, 0.95, SellType.TRAILING_STOP_LOSS, None), + (0.05, 1, SellType.NONE, False, True, -0.06, 1, SellType.TRAILING_STOP_LOSS, + lambda **kwargs: -0.05), + (0.05, 1, SellType.NONE, False, True, 0.09, 1.04, SellType.NONE, + lambda **kwargs: -0.05), + (0.05, 0.95, SellType.NONE, False, True, 0.09, 0.98, SellType.NONE, + lambda current_profit, **kwargs: -0.1 if current_profit < 0.6 else -(current_profit * 2)), + # Error case - static stoploss in place + (0.05, 0.9, SellType.NONE, False, True, 0.09, 0.9, SellType.NONE, + lambda **kwargs: None), + ]) def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, trailing, custom, - profit2, adjusted2, expected2) -> None: + profit2, adjusted2, expected2, custom_stop) -> None: default_conf.update({'strategy': 'DefaultStrategy'}) @@ -319,6 +329,9 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili strategy.trailing_stop = trailing strategy.trailing_stop_positive = -0.05 strategy.custom_stoploss = custom + original_stopvalue = strategy.stoploss_value + if custom_stop: + strategy.stoploss_value = custom_stop now = arrow.utcnow().datetime sl_flag = strategy.stop_loss_reached(current_rate=trade.open_rate * (1 + profit), trade=trade, @@ -342,6 +355,9 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili assert sl_flag.sell_flag is True assert round(trade.stop_loss, 2) == adjusted2 + strategy.stoploss_value = original_stopvalue + + def test_analyze_ticker_default(ohlcv_history, mocker, caplog) -> None: caplog.set_level(logging.DEBUG) From 8574751a07725f5727c30726fa2c7ce996de678a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Dec 2020 10:13:47 +0100 Subject: [PATCH 014/183] Add stoploss_value to strategy template --- docs/strategy-advanced.md | 55 ++++++++++--------- .../subtemplates/strategy_methods_advanced.j2 | 24 ++++++++ 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 79a1f0ec5..e00e3d78d 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -8,6 +8,9 @@ If you're just getting started, please be familiar with the methods described in !!! Note All callback methods described below should only be implemented in a strategy if they are actually used. +!!! Tip + You can get a strategy template containing all below methods by running `freqtrade new-strategy --strategy MyAwesomeStrategy --template advanced` + ## Custom stoploss A stoploss can only ever move upwards - so if you set it to an absolute profit of 2%, you can never move it below this price. @@ -37,32 +40,6 @@ E.g. `current_profit = 0.05` (5% profit) - stoploss returns `0.02` - then you "l The next section will show some examples on what's possible with the custom stoploss function. Of course, many more things are possible, and all examples can be combined at will. -#### Absolute stoploss - -The below example sets absolute profit levels based on the current profit. - -* Use the regular stoploss until 20% profit is reached -* Once profit is > 40%, stoploss will be at 25%, locking in at least 25% of the profit. -* Once profit is > 25% - stoploss will be 15%. -* Once profit is > 20% - stoploss will be set to 7%. - -``` python - custom_stoploss = True - - def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, **kwargs) -> float: - # TODO: Add full docstring here - - # Calculate as `-desired_stop_from_open + current_profit` to get the distance between current_profit and initial price - if current_profit > 0.40: - return (-0.25 + current_profit) - if current_profit > 0.25: - return (-0.15 + current_profit) - if current_profit > 0.20: - return (-0.7 + current_profit) - return 1 -``` - #### Time based trailing stop Use the initial stoploss for the first 60 minutes, after this change to 10% trailing stoploss, and after 2 hours (120 minutes) we use a 5% trailing stoploss. @@ -101,6 +78,32 @@ In this example, we'll trail the highest price with 10% trailing stoploss for `E return -0.15 ``` +#### Absolute stoploss + +The below example sets absolute profit levels based on the current profit. + +* Use the regular stoploss until 20% profit is reached +* Once profit is > 40%, stoploss will be at 25%, locking in at least 25% of the profit. +* Once profit is > 25% - stoploss will be 15%. +* Once profit is > 20% - stoploss will be set to 7%. + +``` python + custom_stoploss = True + + def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> float: + # TODO: Add full docstring here + + # Calculate as `-desired_stop_from_open + current_profit` to get the distance between current_profit and initial price + if current_profit > 0.40: + return (-0.25 + current_profit) + if current_profit > 0.25: + return (-0.15 + current_profit) + if current_profit > 0.20: + return (-0.7 + current_profit) + return 1 +``` + --- ## Custom order timeout rules diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 5ca6e6971..e6ae477b9 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -12,6 +12,30 @@ def bot_loop_start(self, **kwargs) -> None: """ pass +custom_stoploss = True + +def stoploss_value(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, + current_profit: float, **kwargs) -> float: + """ + Custom stoploss logic, returning the new distance relative to current_rate (as ratio). + e.g. returning -0.05 would create a stoploss 5% below current_rate. + The custom stoploss can never be below self.stoploss, which serves as a hard maximum loss. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, returns the initial stoploss value + Only called when custom_stoploss is set to True. + + :param pair: Pair that's about to be sold. + :param trade: trade object. + :param current_time: datetime object, containing the current datetime + :param current_rate: Rate, calculated based on pricing settings in ask_strategy. + :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return float: New stoploss value, relative to the currentrate + """ + return self.stoploss + def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, time_in_force: str, **kwargs) -> bool: """ From 277342f1679f1dea80fa24ed7061424afebf97a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Dec 2020 11:12:22 +0100 Subject: [PATCH 015/183] Rename flag to "use_custom_stoposs" --- docs/strategy-advanced.md | 10 +++++----- freqtrade/strategy/interface.py | 6 +++--- .../subtemplates/strategy_methods_advanced.j2 | 4 ++-- tests/strategy/test_interface.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index e00e3d78d..f2d8e4151 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -16,12 +16,12 @@ If you're just getting started, please be familiar with the methods described in A stoploss can only ever move upwards - so if you set it to an absolute profit of 2%, you can never move it below this price. Also, the traditional `stoploss` value serves as an absolute lower level and will be instated as the initial stoploss. -The usage of the custom stoploss method must be enabled by setting `custom_stoploss=True` on the strategy object. +The usage of the custom stoploss method must be enabled by setting `use_custom_stoploss=True` on the strategy object. The method must return a stoploss value (float / number) with a relative ratio below the current price. E.g. `current_profit = 0.05` (5% profit) - stoploss returns `0.02` - then you "locked in" a profit of 3% (`0.05 - 0.02 = 0.03`). ``` python - custom_stoploss = True + use_custom_stoploss = True def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs) -> float: @@ -45,7 +45,7 @@ Of course, many more things are possible, and all examples can be combined at wi Use the initial stoploss for the first 60 minutes, after this change to 10% trailing stoploss, and after 2 hours (120 minutes) we use a 5% trailing stoploss. ``` python - custom_stoploss = True + use_custom_stoploss = True def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs) -> float: @@ -65,7 +65,7 @@ Use a different stoploss depending on the pair. In this example, we'll trail the highest price with 10% trailing stoploss for `ETH/BTC` and `XRP/BTC`, with 5% trailing stoploss for `LTC/BTC` and with 15% for all other pairs. ``` python - custom_stoploss = True + use_custom_stoploss = True def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs) -> float: @@ -88,7 +88,7 @@ The below example sets absolute profit levels based on the current profit. * Once profit is > 20% - stoploss will be set to 7%. ``` python - custom_stoploss = True + use_custom_stoploss = True def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs) -> float: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 9378f1996..d93dda849 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -89,7 +89,7 @@ class IStrategy(ABC): trailing_stop_positive: Optional[float] = None trailing_stop_positive_offset: float = 0.0 trailing_only_offset_is_reached = False - custom_stoploss: bool = False + use_custom_stoploss: bool = False # associated timeframe ticker_interval: str # DEPRECATED @@ -265,7 +265,7 @@ class IStrategy(ABC): For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ When not implemented by a strategy, returns the initial stoploss value - Only called when custom_stoploss is set to True. + Only called when use_custom_stoploss is set to True. :param pair: Pair that's about to be sold. :param trade: trade object. @@ -554,7 +554,7 @@ class IStrategy(ABC): # Initiate stoploss with open_rate. Does nothing if stoploss is already set. trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True) - if self.custom_stoploss: + if self.use_custom_stoploss: stop_loss_value = strategy_safe_wrapper(self.stoploss_value, default_retval=None )(pair=trade.pair, trade=trade, current_time=current_time, diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index e6ae477b9..0ae3e077c 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -12,7 +12,7 @@ def bot_loop_start(self, **kwargs) -> None: """ pass -custom_stoploss = True +use_custom_stoploss = True def stoploss_value(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, current_profit: float, **kwargs) -> float: @@ -24,7 +24,7 @@ def stoploss_value(self, pair: str, trade: 'Trade', current_time: 'datetime', cu For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ When not implemented by a strategy, returns the initial stoploss value - Only called when custom_stoploss is set to True. + Only called when use_custom_stoploss is set to True. :param pair: Pair that's about to be sold. :param trade: trade object. diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index e8a3ee8de..5922ad9b2 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -328,7 +328,7 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili trade.adjust_min_max_rates(trade.open_rate) strategy.trailing_stop = trailing strategy.trailing_stop_positive = -0.05 - strategy.custom_stoploss = custom + strategy.use_custom_stoploss = custom original_stopvalue = strategy.stoploss_value if custom_stop: strategy.stoploss_value = custom_stop From 9d5961e2247295d086023cacc9d959722404d0f7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Dec 2020 11:17:50 +0100 Subject: [PATCH 016/183] Rename method to custom_stoploss --- docs/strategy-advanced.md | 16 ++++++++-------- freqtrade/strategy/interface.py | 6 +++--- .../subtemplates/strategy_methods_advanced.j2 | 4 ++-- tests/strategy/test_default_strategy.py | 8 +++++--- tests/strategy/test_interface.py | 11 +++++------ 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index f2d8e4151..bb455ff9f 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -23,8 +23,8 @@ E.g. `current_profit = 0.05` (5% profit) - stoploss returns `0.02` - then you "l ``` python use_custom_stoploss = True - def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, **kwargs) -> float: + def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> float: # TODO: Add full docstring here return 0.04 ``` @@ -47,8 +47,8 @@ Use the initial stoploss for the first 60 minutes, after this change to 10% trai ``` python use_custom_stoploss = True - def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, **kwargs) -> float: + def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> float: # TODO: Add full docstring here # Make sure you have the longest interval first - these conditions are evaluated from top to bottom. @@ -67,8 +67,8 @@ In this example, we'll trail the highest price with 10% trailing stoploss for `E ``` python use_custom_stoploss = True - def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, **kwargs) -> float: + def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> float: # TODO: Add full docstring here if pair in ('ETH/BTC', 'XRP/BTC'): @@ -90,8 +90,8 @@ The below example sets absolute profit levels based on the current profit. ``` python use_custom_stoploss = True - def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, **kwargs) -> float: + def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> float: # TODO: Add full docstring here # Calculate as `-desired_stop_from_open + current_profit` to get the distance between current_profit and initial price diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index d93dda849..61c879e8f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -255,8 +255,8 @@ class IStrategy(ABC): """ return True - def stoploss_value(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, - current_profit: float, **kwargs) -> float: + def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> float: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). e.g. returning -0.05 would create a stoploss 5% below current_rate. @@ -555,7 +555,7 @@ class IStrategy(ABC): trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True) if self.use_custom_stoploss: - stop_loss_value = strategy_safe_wrapper(self.stoploss_value, default_retval=None + stop_loss_value = strategy_safe_wrapper(self.custom_stoploss, default_retval=None )(pair=trade.pair, trade=trade, current_time=current_time, current_rate=current_rate, diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 0ae3e077c..f4cda2e89 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -14,8 +14,8 @@ def bot_loop_start(self, **kwargs) -> None: use_custom_stoploss = True -def stoploss_value(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, - current_profit: float, **kwargs) -> float: +def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, + current_profit: float, **kwargs) -> float: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). e.g. returning -0.05 would create a stoploss 5% below current_rate. diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py index c5d76b4c5..ec7b3c33d 100644 --- a/tests/strategy/test_default_strategy.py +++ b/tests/strategy/test_default_strategy.py @@ -1,7 +1,9 @@ from datetime import datetime -from freqtrade.persistence.models import Trade + from pandas import DataFrame +from freqtrade.persistence.models import Trade + from .strats.default_strategy import DefaultStrategy @@ -38,5 +40,5 @@ def test_default_strategy(result, fee): assert strategy.confirm_trade_exit(pair='ETH/BTC', trade=trade, order_type='limit', amount=0.1, rate=20000, time_in_force='gtc', sell_reason='roi') is True - assert strategy.stoploss_value(pair='ETH/BTC', trade=trade, current_time=datetime.now(), - current_rate=20_000, current_profit=0.05) == strategy.stoploss + assert strategy.custom_stoploss(pair='ETH/BTC', trade=trade, current_time=datetime.now(), + current_rate=20_000, current_profit=0.05) == strategy.stoploss diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 5922ad9b2..7eed43302 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -1,5 +1,4 @@ # pragma pylint: disable=missing-docstring, C0103 -from freqtrade.strategy.interface import SellCheckTuple, SellType import logging from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock @@ -11,9 +10,10 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import load_data -from freqtrade.exceptions import OperationalException, StrategyError +from freqtrade.exceptions import StrategyError from freqtrade.persistence import PairLocks, Trade from freqtrade.resolvers import StrategyResolver +from freqtrade.strategy.interface import SellCheckTuple, SellType from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from tests.conftest import log_has, log_has_re @@ -329,9 +329,9 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili strategy.trailing_stop = trailing strategy.trailing_stop_positive = -0.05 strategy.use_custom_stoploss = custom - original_stopvalue = strategy.stoploss_value + original_stopvalue = strategy.custom_stoploss if custom_stop: - strategy.stoploss_value = custom_stop + strategy.custom_stoploss = custom_stop now = arrow.utcnow().datetime sl_flag = strategy.stop_loss_reached(current_rate=trade.open_rate * (1 + profit), trade=trade, @@ -355,8 +355,7 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili assert sl_flag.sell_flag is True assert round(trade.stop_loss, 2) == adjusted2 - strategy.stoploss_value = original_stopvalue - + strategy.custom_stoploss = original_stopvalue def test_analyze_ticker_default(ohlcv_history, mocker, caplog) -> None: From 676dd0d664e9ee59f2008b518eb12795e3fe9d85 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Dec 2020 11:22:15 +0100 Subject: [PATCH 017/183] Improve documentation --- docs/stoploss.md | 1 + docs/strategy-advanced.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/stoploss.md b/docs/stoploss.md index 1e21fc50d..671e643b0 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -78,6 +78,7 @@ At this stage the bot contains the following stoploss support modes: 2. Trailing stop loss. 3. Trailing stop loss, custom positive loss. 4. Trailing stop loss only once the trade has reached a certain offset. +5. [Custom stoploss function](strategy-advanced.md#custom-stoploss) ### Static Stop Loss diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index bb455ff9f..4833fbade 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -29,6 +29,8 @@ E.g. `current_profit = 0.05` (5% profit) - stoploss returns `0.02` - then you "l return 0.04 ``` +Stoploss on exchange works similar to trailing_stop, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchange-freqtrade)). + !!! Note "Use of dates" All time-based calculations should be done based on `current_time` - using `datetime.now()` or `datetime.utcnow()` is discouraged, as this will break backtesting support. From fc0d14c1b5c9b15ef04d5b9abb32283bc38bd78c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Dec 2020 19:14:18 +0100 Subject: [PATCH 018/183] Improve documentation --- docs/strategy-advanced.md | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 4833fbade..49720d729 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -20,16 +20,35 @@ The usage of the custom stoploss method must be enabled by setting `use_custom_s The method must return a stoploss value (float / number) with a relative ratio below the current price. E.g. `current_profit = 0.05` (5% profit) - stoploss returns `0.02` - then you "locked in" a profit of 3% (`0.05 - 0.02 = 0.03`). +To simulate a regular trailing stoploss of 4% (trailing 4% behind the maximum reached price) you would use the following very simple method: + ``` python use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs) -> float: - # TODO: Add full docstring here - return 0.04 + """ + Custom stoploss logic, returning the new distance relative to current_rate (as ratio). + e.g. returning -0.05 would create a stoploss 5% below current_rate. + The custom stoploss can never be below self.stoploss, which serves as a hard maximum loss. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, returns the initial stoploss value + Only called when use_custom_stoploss is set to True. + + :param pair: Pair that's about to be sold. + :param trade: trade object. + :param current_time: datetime object, containing the current datetime + :param current_rate: Rate, calculated based on pricing settings in ask_strategy. + :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return float: New stoploss value, relative to the currentrate + """ + return -0.04 ``` -Stoploss on exchange works similar to trailing_stop, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchange-freqtrade)). +Stoploss on exchange works similar to `trailing_stop`, and the stoploss on exchange is updated as configured in `stoploss_on_exchange_interval` ([More details about stoploss on exchange](stoploss.md#stop-loss-on-exchange-freqtrade)). !!! Note "Use of dates" All time-based calculations should be done based on `current_time` - using `datetime.now()` or `datetime.utcnow()` is discouraged, as this will break backtesting support. @@ -51,7 +70,6 @@ Use the initial stoploss for the first 60 minutes, after this change to 10% trai def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs) -> float: - # TODO: Add full docstring here # Make sure you have the longest interval first - these conditions are evaluated from top to bottom. if current_time - timedelta(minutes=120) > trade.open_date: @@ -71,7 +89,6 @@ In this example, we'll trail the highest price with 10% trailing stoploss for `E def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs) -> float: - # TODO: Add full docstring here if pair in ('ETH/BTC', 'XRP/BTC'): return -0.10 @@ -94,7 +111,6 @@ The below example sets absolute profit levels based on the current profit. def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, **kwargs) -> float: - # TODO: Add full docstring here # Calculate as `-desired_stop_from_open + current_profit` to get the distance between current_profit and initial price if current_profit > 0.40: From 092ebf845d2e27a80d0f2cdcf3c55c771b6f5742 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Dec 2020 05:37:16 +0000 Subject: [PATCH 019/183] Bump scikit-learn from 0.23.2 to 0.24.0 Bumps [scikit-learn](https://github.com/scikit-learn/scikit-learn) from 0.23.2 to 0.24.0. - [Release notes](https://github.com/scikit-learn/scikit-learn/releases) - [Commits](https://github.com/scikit-learn/scikit-learn/compare/0.23.2...0.24.0) Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index c51062bf7..a2446ddb8 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -3,7 +3,7 @@ # Required for hyperopt scipy==1.5.4 -scikit-learn==0.23.2 +scikit-learn==0.24.0 scikit-optimize==0.8.1 filelock==3.0.12 joblib==1.0.0 From 30087697e06c5e64309e891b8ab2c06f78246c10 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Dec 2020 08:21:17 +0000 Subject: [PATCH 020/183] Bump pandas from 1.1.5 to 1.2.0 Bumps [pandas](https://github.com/pandas-dev/pandas) from 1.1.5 to 1.2.0. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/master/RELEASE.md) - [Commits](https://github.com/pandas-dev/pandas/compare/v1.1.5...v1.2.0) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 594c22b74..bab13ed03 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy==1.19.4 -pandas==1.1.5 +pandas==1.2.0 ccxt==1.39.79 aiohttp==3.7.3 From 0d4cf32086f3d9942fb2ae0517da464b5b517457 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Dec 2020 09:50:48 +0100 Subject: [PATCH 021/183] Slightly adapt to pandas incompatibility --- freqtrade/data/btanalysis.py | 2 +- tests/rpc/test_rpc_apiserver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 513fba9e7..2b51f5371 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -347,7 +347,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, # Resample to timeframe to make sure trades match candles _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date' )[['profit_percent']].sum() - df.loc[:, col_name] = _trades_sum.cumsum() + df.loc[:, col_name] = _trades_sum['profit_percent'].cumsum() # Set first value to 0 df.loc[df.iloc[0].name, col_name] = 0 # FFill to get continuous diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index e7eee6f05..5e608fb25 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -929,7 +929,7 @@ def test_api_pair_candles(botclient, ohlcv_history): ['2017-11-26 08:55:00', 8.88e-05, 8.942e-05, 8.88e-05, 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0, 1511686500000, 8.88e-05, None], ['2017-11-26 09:00:00', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, - 0.7039405, 8.885000000000002e-05, 0, 0, 1511686800000, None, None] + 0.7039405, 8.885e-05, 0, 0, 1511686800000, None, None] ]) From 003552d78c3ef426f42bcb132645842b763be569 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Dec 2020 10:19:24 +0100 Subject: [PATCH 022/183] Remove custom header section from docs --- docs/partials/header.html | 54 --------------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 docs/partials/header.html diff --git a/docs/partials/header.html b/docs/partials/header.html deleted file mode 100644 index 32202bccc..000000000 --- a/docs/partials/header.html +++ /dev/null @@ -1,54 +0,0 @@ -
- - - - -
From accc59aa1b90efbb7a1784da9da2ecc6739a88f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Dec 2020 13:49:08 +0100 Subject: [PATCH 023/183] Reinstate jquery --- docs/overrides/main.html | 8 ++++++++ mkdocs.yml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 docs/overrides/main.html diff --git a/docs/overrides/main.html b/docs/overrides/main.html new file mode 100644 index 000000000..916d26770 --- /dev/null +++ b/docs/overrides/main.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} + +{% block site_meta %} + + + +{% endblock %} diff --git a/mkdocs.yml b/mkdocs.yml index a7ae0cc96..a0b5d8641 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -38,7 +38,7 @@ nav: theme: name: material logo: 'images/logo.png' - custom_dir: 'docs' + custom_dir: 'docs/overrides' palette: primary: 'blue grey' accent: 'tear' From ecea6c95263bbedc790c62fd0258009321cd64fb Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Dec 2020 14:02:30 +0100 Subject: [PATCH 024/183] Move jquery to the bottom --- docs/overrides/main.html | 4 ++-- mkdocs.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/overrides/main.html b/docs/overrides/main.html index 916d26770..910af0973 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -1,8 +1,8 @@ {% extends "base.html" %} -{% block site_meta %} +{% block scripts %} - {% endblock %} diff --git a/mkdocs.yml b/mkdocs.yml index a0b5d8641..a14c67b03 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -38,6 +38,7 @@ nav: theme: name: material logo: 'images/logo.png' + favicon: 'images/logo.png' custom_dir: 'docs/overrides' palette: primary: 'blue grey' From dcc7d559ee6edbf82b45e5f1596f2c1281ae5c35 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 28 Dec 2020 14:08:57 +0100 Subject: [PATCH 025/183] Reinstate header partials --- docs/overrides/main.html | 8 ------ docs/partials/header.html | 51 +++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 2 +- 3 files changed, 52 insertions(+), 9 deletions(-) delete mode 100644 docs/overrides/main.html create mode 100644 docs/partials/header.html diff --git a/docs/overrides/main.html b/docs/overrides/main.html deleted file mode 100644 index 910af0973..000000000 --- a/docs/overrides/main.html +++ /dev/null @@ -1,8 +0,0 @@ -{% extends "base.html" %} - -{% block scripts %} - - - -{% endblock %} diff --git a/docs/partials/header.html b/docs/partials/header.html new file mode 100644 index 000000000..f5243225b --- /dev/null +++ b/docs/partials/header.html @@ -0,0 +1,51 @@ +{#- +This file was automatically generated - do not edit +-#} +{% set site_url = config.site_url | d(nav.homepage.url, true) | url %} +{% if not config.use_directory_urls and site_url[0] == site_url[-1] == "." %} +{% set site_url = site_url ~ "/index.html" %} +{% endif %} +
+ + + + +
diff --git a/mkdocs.yml b/mkdocs.yml index a14c67b03..96cfa7651 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -39,7 +39,7 @@ theme: name: material logo: 'images/logo.png' favicon: 'images/logo.png' - custom_dir: 'docs/overrides' + custom_dir: 'docs' palette: primary: 'blue grey' accent: 'tear' From 238e9aabb1788b42d096ceb1a5c24d49dbadb1a5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Dec 2020 20:05:07 +0100 Subject: [PATCH 026/183] Add test showing wrong behaviour --- tests/exchange/test_kraken.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/exchange/test_kraken.py b/tests/exchange/test_kraken.py index 3803658eb..97f428e2f 100644 --- a/tests/exchange/test_kraken.py +++ b/tests/exchange/test_kraken.py @@ -89,6 +89,7 @@ def test_get_balances_prod(default_conf, mocker): '2ST': balance_item.copy(), '3ST': balance_item.copy(), '4ST': balance_item.copy(), + 'EUR': balance_item.copy(), }) kraken_open_orders = [{'symbol': '1ST/EUR', 'type': 'limit', @@ -123,21 +124,22 @@ def test_get_balances_prod(default_conf, mocker): 'remaining': 2.0, }, {'status': 'open', - 'symbol': 'BTC/3ST', + 'symbol': '3ST/EUR', 'type': 'limit', 'side': 'buy', - 'price': 20, + 'price': 0.02, 'cost': 0.0, - 'amount': 3.0, + 'amount': 100.0, 'filled': 0.0, 'average': 0.0, - 'remaining': 3.0, + 'remaining': 100.0, }] api_mock.fetch_open_orders = MagicMock(return_value=kraken_open_orders) default_conf['dry_run'] = False exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken") balances = exchange.get_balances() - assert len(balances) == 4 + assert len(balances) == 5 + assert balances['1ST']['free'] == 9.0 assert balances['1ST']['total'] == 10.0 assert balances['1ST']['used'] == 1.0 @@ -146,13 +148,17 @@ def test_get_balances_prod(default_conf, mocker): assert balances['2ST']['total'] == 10.0 assert balances['2ST']['used'] == 4.0 - assert balances['3ST']['free'] == 7.0 + assert balances['3ST']['free'] == 10.0 assert balances['3ST']['total'] == 10.0 - assert balances['3ST']['used'] == 3.0 + assert balances['3ST']['used'] == 0.0 assert balances['4ST']['free'] == 10.0 assert balances['4ST']['total'] == 10.0 assert balances['4ST']['used'] == 0.0 + + assert balances['EUR']['free'] == 8.0 + assert balances['EUR']['total'] == 10.0 + assert balances['EUR']['used'] == 2.0 ccxt_exceptionhandlers(mocker, default_conf, api_mock, "kraken", "get_balances", "fetch_balance") From b607740dd10d41394985715a70393a2ddc261dac Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Dec 2020 20:06:37 +0100 Subject: [PATCH 027/183] Fix kraken balance bug if open buy orders exist --- freqtrade/exchange/kraken.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 6dbb751e5..724b11189 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -48,7 +48,7 @@ class Kraken(Exchange): orders = self._api.fetch_open_orders() order_list = [(x["symbol"].split("/")[0 if x["side"] == "sell" else 1], - x["remaining"], + x["remaining"] if x["side"] == "sell" else x["remaining"] * x["price"], # Don't remove the below comment, this can be important for debuggung # x["side"], x["amount"], ) for x in orders] From b8899b39ec2aacd7be38de76edc336520e72dfe1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Dec 2020 06:29:59 +0100 Subject: [PATCH 028/183] Show advanced plot-config section again closes #4132 --- docs/plotting.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/plotting.md b/docs/plotting.md index ed682e44b..19ddb4f57 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -208,6 +208,7 @@ Sample configuration with inline comments explaining the process: } ``` + !!! Note The above configuration assumes that `ema10`, `ema50`, `senkou_a`, `senkou_b`, `macd`, `macdsignal`, `macdhist` and `rsi` are columns in the DataFrame created by the strategy. From 2fdda8e448a5147f0abef0ba0fe26911909e8572 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Dec 2020 08:30:41 +0100 Subject: [PATCH 029/183] plot-profit should fail gracefully if no trade is within the selected timerange closes #4119 --- freqtrade/plot/plotting.py | 2 ++ tests/test_plotting.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 497218deb..40e3da9c9 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -444,6 +444,8 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame], # Trim trades to available OHLCV data trades = extract_trades_of_period(df_comb, trades, date_index=True) + if len(trades) == 0: + raise OperationalException('No trades found in selected timerange.') # Add combined cumulative profit df_comb = create_cum_profit(df_comb, trades, 'cum_profit', timeframe) diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 42847ca50..8e7b0ef7c 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -353,6 +353,10 @@ def test_generate_profit_graph(testdatadir): profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}") assert isinstance(profit_pair, go.Scatter) + with pytest.raises(OperationalException, match=r"No trades found.*"): + # Pair cannot be empty - so it's an empty dataframe. + generate_profit_graph(pairs, data, trades.loc[trades['pair'].isnull()], timeframe="5m") + def test_start_plot_dataframe(mocker): aup = mocker.patch("freqtrade.plot.plotting.load_and_plot_trades", MagicMock()) From 704cf143835fb6bb1e12fb2a3b611415894f0126 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Dec 2020 09:55:44 +0100 Subject: [PATCH 030/183] Add expand_pairlist method --- freqtrade/exchange/exchange.py | 2 +- .../plugins/pairlist/pairlist_helpers.py | 18 +++++++++++ freqtrade/plugins/pairlistmanager.py | 10 +++++- tests/plugins/test_pairlist.py | 32 +++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 freqtrade/plugins/pairlist/pairlist_helpers.py diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 6f495e605..11a0ef8e6 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -208,7 +208,7 @@ class Exchange: return self._api.precisionMode def get_markets(self, base_currencies: List[str] = None, quote_currencies: List[str] = None, - pairs_only: bool = False, active_only: bool = False) -> Dict: + pairs_only: bool = False, active_only: bool = False) -> Dict[str, Any]: """ Return exchange ccxt markets, filtered out by base currency and quote currency if this was requested in parameters. diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py new file mode 100644 index 000000000..7d365a344 --- /dev/null +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -0,0 +1,18 @@ +import re +from typing import List + + +def expand_pairlist(wildcardpl: List[str], available_pairs: List[str]) -> List[str]: + """ + TODO: Add docstring here + """ + result = [] + for pair_wc in wildcardpl: + try: + comp = re.compile(pair_wc) + result += [ + pair for pair in available_pairs if re.match(comp, pair) + ] + except re.error as err: + raise ValueError(f"Wildcard error in {pair_wc}, {err}") + return result diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index b71f02898..ea1f4ecc7 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -1,6 +1,7 @@ """ PairList manager class """ +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist import logging from copy import deepcopy from typing import Any, Dict, List @@ -55,6 +56,13 @@ class PairListManager(): """ return self._blacklist + @property + def expanded_blacklist(self) -> List[str]: + """ + Has the expanded blacklist (including wildcard expansion) + """ + return expand_pairlist(self._blacklist, self._exchange.get_markets().keys()) + @property def name_list(self) -> List[str]: """ @@ -121,7 +129,7 @@ class PairListManager(): :return: pairlist - blacklisted pairs """ for pair in deepcopy(pairlist): - if pair in self._blacklist: + if pair in self.expanded_blacklist: logmethod(f"Pair {pair} in your blacklist. Removing it from whitelist...") pairlist.remove(pair) return pairlist diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 1795fc27f..25597ef93 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -6,6 +6,7 @@ import pytest from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver from tests.conftest import get_patched_freqtradebot, log_has, log_has_re @@ -804,3 +805,34 @@ def test_performance_filter(mocker, whitelist_conf, pairlists, pair_allowlist, o freqtrade.pairlists.refresh_pairlist() allowlist = freqtrade.pairlists.whitelist assert allowlist == allowlist_result + + +@pytest.mark.parametrize('wildcardlist,pairs,expected', [ + (['BTC/USDT'], + ['BTC/USDT'], + ['BTC/USDT']), + (['BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETH/USDT']), + (['BTC/USDT', 'ETH/USDT'], + ['BTC/USDT'], ['BTC/USDT']), # Test one too many + (['.*/USDT'], + ['BTC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETH/USDT']), # Wildcard simple + (['.*C/USDT'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETC/USDT']), # Wildcard exclude one + (['.*UP/USDT', 'BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'], + ['BTC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT']), # Wildcard exclude one + (['BTC/.*', 'ETH/.*'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP'], + ['BTC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP']), # Wildcard exclude one + (['*UP/USDT', 'BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'], + None), +]) +def test_expand_pairlist(wildcardlist, pairs, expected): + if expected is None: + with pytest.raises(ValueError, match=r'Wildcard error in \*UP/USDT,'): + expand_pairlist(wildcardlist, pairs) + else: + assert sorted(expand_pairlist(wildcardlist, pairs)) == sorted(expected) From 9feabe707fc501086a0cfee59799be8d6296f62f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Dec 2020 09:57:31 +0100 Subject: [PATCH 031/183] Fix RPC methods to allow wildcards (and validate wildcards) --- freqtrade/rpc/rpc.py | 17 +++++++++-------- tests/rpc/test_rpc.py | 14 ++++++++++++-- tests/rpc/test_rpc_apiserver.py | 13 +++++++++++++ tests/rpc/test_rpc_telegram.py | 12 +++++------- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 42ab76622..70a99a186 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -20,6 +20,7 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.loggers import bufferHandler from freqtrade.misc import shorten_date from freqtrade.persistence import PairLocks, Trade +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State from freqtrade.strategy.interface import SellType @@ -673,23 +674,23 @@ class RPC: """ Returns the currently active blacklist""" errors = {} if add: - stake_currency = self._freqtrade.config.get('stake_currency') for pair in add: - if self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency: - if pair not in self._freqtrade.pairlists.blacklist: + if pair not in self._freqtrade.pairlists.blacklist: + try: + expand_pairlist([pair], self._freqtrade.exchange.get_markets().keys()) self._freqtrade.pairlists.blacklist.append(pair) - else: - errors[pair] = { - 'error_msg': f'Pair {pair} already in pairlist.'} + except ValueError: + errors[pair] = { + 'error_msg': f'Pair {pair} is not a valid wildcard.'} else: errors[pair] = { - 'error_msg': f"Pair {pair} does not match stake currency." - } + 'error_msg': f'Pair {pair} already in pairlist.'} res = {'method': self._freqtrade.pairlists.name_list, 'length': len(self._freqtrade.pairlists.blacklist), 'blacklist': self._freqtrade.pairlists.blacklist, + 'blacklist_expanded': self._freqtrade.pairlists.expanded_blacklist, 'errors': errors, } return res diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 19788c067..8ec356d54 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -957,14 +957,24 @@ def test_rpc_blacklist(mocker, default_conf) -> None: assert isinstance(ret['errors'], dict) assert ret['errors']['ETH/BTC']['error_msg'] == 'Pair ETH/BTC already in pairlist.' - ret = rpc._rpc_blacklist(["ETH/ETH"]) + ret = rpc._rpc_blacklist(["*/BTC"]) assert 'StaticPairList' in ret['method'] assert len(ret['blacklist']) == 3 assert ret['blacklist'] == default_conf['exchange']['pair_blacklist'] assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC', 'ETH/BTC'] + assert ret['blacklist_expanded'] == ['ETH/BTC'] + assert 'errors' in ret + assert isinstance(ret['errors'], dict) + assert ret['errors'] == {'*/BTC': {'error_msg': 'Pair */BTC is not a valid wildcard.'}} + + ret = rpc._rpc_blacklist(["XRP/.*"]) + assert 'StaticPairList' in ret['method'] + assert len(ret['blacklist']) == 4 + assert ret['blacklist'] == default_conf['exchange']['pair_blacklist'] + assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC', 'ETH/BTC', 'XRP/.*'] + assert ret['blacklist_expanded'] == ['ETH/BTC', 'XRP/BTC'] assert 'errors' in ret assert isinstance(ret['errors'], dict) - assert ret['errors']['ETH/ETH']['error_msg'] == 'Pair ETH/ETH does not match stake currency.' def test_rpc_edge_disabled(mocker, default_conf) -> None: diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 5e608fb25..8da4ebfe7 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -730,7 +730,9 @@ def test_api_blacklist(botclient, mocker): rc = client_get(client, f"{BASE_URI}/blacklist") assert_response(rc) + # DOGE and HOT are not in the markets mock! assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC"], + "blacklist_expanded": [], "length": 2, "method": ["StaticPairList"], "errors": {}, @@ -741,11 +743,22 @@ def test_api_blacklist(botclient, mocker): data='{"blacklist": ["ETH/BTC"]}') assert_response(rc) assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC"], + "blacklist_expanded": ["ETH/BTC"], "length": 3, "method": ["StaticPairList"], "errors": {}, } + rc = client_post(client, f"{BASE_URI}/blacklist", + data='{"blacklist": ["XRP/.*"]}') + assert_response(rc) + assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"], + "blacklist_expanded": ["ETH/BTC", "XRP/BTC"], + "length": 4, + "method": ["StaticPairList"], + "errors": {}, + } + def test_api_whitelist(botclient): ftbot, client = botclient diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 97b9e5e7c..a21a19e3a 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1011,15 +1011,13 @@ def test_blacklist_static(default_conf, update, mocker) -> None: msg_mock.reset_mock() context = MagicMock() - context.args = ["ETH/ETH"] + context.args = ["XRP/.*"] telegram._blacklist(update=update, context=context) - assert msg_mock.call_count == 2 - assert ("Error adding `ETH/ETH` to blacklist: `Pair ETH/ETH does not match stake currency.`" - in msg_mock.call_args_list[0][0][0]) + assert msg_mock.call_count == 1 - assert ("Blacklist contains 3 pairs\n`DOGE/BTC, HOT/BTC, ETH/BTC`" - in msg_mock.call_args_list[1][0][0]) - assert freqtradebot.pairlists.blacklist == ["DOGE/BTC", "HOT/BTC", "ETH/BTC"] + assert ("Blacklist contains 4 pairs\n`DOGE/BTC, HOT/BTC, ETH/BTC, XRP/.*`" + in msg_mock.call_args_list[0][0][0]) + assert freqtradebot.pairlists.blacklist == ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"] def test_telegram_logs(default_conf, update, mocker) -> None: From 0affacd39aa808d60d130b0518eaf698c53399ae Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Dec 2020 10:14:22 +0100 Subject: [PATCH 032/183] Support invalid regex blacklist from config --- freqtrade/plugins/pairlist/pairlist_helpers.py | 7 ++++++- freqtrade/plugins/pairlistmanager.py | 7 ++++++- tests/plugins/test_pairlist.py | 17 +++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py index 7d365a344..0a0812d6a 100644 --- a/freqtrade/plugins/pairlist/pairlist_helpers.py +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -4,7 +4,12 @@ from typing import List def expand_pairlist(wildcardpl: List[str], available_pairs: List[str]) -> List[str]: """ - TODO: Add docstring here + Expand pairlist potentially containing wildcards based on available markets. + This will implicitly filter all pairs in the wildcard-list which are not in available_pairs. + :param wildcardpl: List of Pairlists, which may contain regex + :param available_pairs: List of all available pairs, usually with `exchange.get_markets().keys()` + :return expanded pairlist, with Regexes from wildcardpl applied to match all available pairs. + :raises: ValueError if a wildcard is invalid (like '*/BTC' - which should be `.*/BTC`) """ result = [] for pair_wc in wildcardpl: diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index ea1f4ecc7..a0b8c63bc 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -128,8 +128,13 @@ class PairListManager(): :param logmethod: Function that'll be called, `logger.info` or `logger.warning`. :return: pairlist - blacklisted pairs """ + try: + blacklist = self.expanded_blacklist + except ValueError as err: + logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") + return [] for pair in deepcopy(pairlist): - if pair in self.expanded_blacklist: + if pair in blacklist: logmethod(f"Pair {pair} in your blacklist. Removing it from whitelist...") pairlist.remove(pair) return pairlist diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 25597ef93..d822f8319 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -156,6 +156,23 @@ def test_refresh_static_pairlist(mocker, markets, static_pl_conf): assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist +def test_invalid_blacklist(mocker, markets, static_pl_conf, caplog): + static_pl_conf['exchange']['pair_blacklist'] = ['*/BTC'] + freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + markets=PropertyMock(return_value=markets), + ) + freqtrade.pairlists.refresh_pairlist() + # List ordered by BaseVolume + whitelist = [] + # Ensure all except those in whitelist are removed + assert set(whitelist) == set(freqtrade.pairlists.whitelist) + assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist + log_has_re(r"Pair blacklist contains an invalid Wildcard.*", caplog) + + def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf): mocker.patch.multiple( From 04624aae40d9205c56e5b07547eb427eeab540f7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 30 Dec 2020 10:21:05 +0100 Subject: [PATCH 033/183] Add documentation for wildcard-blacklist --- docs/includes/pairlists.md | 8 ++++++++ freqtrade/plugins/pairlist/pairlist_helpers.py | 2 +- freqtrade/plugins/pairlistmanager.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 732dfa5bb..82e578484 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -10,6 +10,14 @@ If multiple Pairlist Handlers are used, they are chained and a combination of al Inactive markets are always removed from the resulting pairlist. Explicitly blacklisted pairs (those in the `pair_blacklist` configuration setting) are also always removed from the resulting pairlist. +### Pair blacklist + +The pair blacklist (configured via `exchange.pair_blacklist` in the configuration) disallows certain pairs from trading. +This can be as simple as excluding `DOGE/BTC` - which will remove exactly this pair. + +The pair-blacklist does also support wildcards (in regex-style) - so `BNB/.*` will exclude ALL pairs that start with BNB. +You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged tokens (check Naming conventions for your exchange!) + ### Available Pairlist Handlers * [`StaticPairList`](#static-pair-list) (default, if not configured differently) diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py index 0a0812d6a..3352777f0 100644 --- a/freqtrade/plugins/pairlist/pairlist_helpers.py +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -7,7 +7,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str]) -> List[s Expand pairlist potentially containing wildcards based on available markets. This will implicitly filter all pairs in the wildcard-list which are not in available_pairs. :param wildcardpl: List of Pairlists, which may contain regex - :param available_pairs: List of all available pairs, usually with `exchange.get_markets().keys()` + :param available_pairs: List of all available pairs (`exchange.get_markets().keys()`) :return expanded pairlist, with Regexes from wildcardpl applied to match all available pairs. :raises: ValueError if a wildcard is invalid (like '*/BTC' - which should be `.*/BTC`) """ diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index a0b8c63bc..867c07736 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -1,7 +1,6 @@ """ PairList manager class """ -from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist import logging from copy import deepcopy from typing import Any, Dict, List @@ -11,6 +10,7 @@ from cachetools import TTLCache, cached from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import PairListResolver From bd7600ff0673137a1377d0ac704c180f1dc707bb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Dec 2020 09:43:24 +0100 Subject: [PATCH 034/183] Small visual changes --- docs/includes/pairlists.md | 2 +- freqtrade/plugins/pairlistmanager.py | 22 ++++++---------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 82e578484..8919c4e3d 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -16,7 +16,7 @@ The pair blacklist (configured via `exchange.pair_blacklist` in the configuratio This can be as simple as excluding `DOGE/BTC` - which will remove exactly this pair. The pair-blacklist does also support wildcards (in regex-style) - so `BNB/.*` will exclude ALL pairs that start with BNB. -You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged tokens (check Naming conventions for your exchange!) +You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged tokens (check Pair naming conventions for your exchange!) ### Available Pairlist Handlers diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 867c07736..ad7b46cb8 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -43,37 +43,29 @@ class PairListManager(): @property def whitelist(self) -> List[str]: - """ - Has the current whitelist - """ + """The current whitelist""" return self._whitelist @property def blacklist(self) -> List[str]: """ - Has the current blacklist + The current blacklist -> no need to overwrite in subclasses """ return self._blacklist @property def expanded_blacklist(self) -> List[str]: - """ - Has the expanded blacklist (including wildcard expansion) - """ + """The expanded blacklist (including wildcard expansion)""" return expand_pairlist(self._blacklist, self._exchange.get_markets().keys()) @property def name_list(self) -> List[str]: - """ - Get list of loaded Pairlist Handler names - """ + """Get list of loaded Pairlist Handler names""" return [p.name for p in self._pairlist_handlers] def short_desc(self) -> List[Dict]: - """ - List of short_desc for each Pairlist Handler - """ + """List of short_desc for each Pairlist Handler""" return [{p.name: p.short_desc()} for p in self._pairlist_handlers] @cached(TTLCache(maxsize=1, ttl=1800)) @@ -81,9 +73,7 @@ class PairListManager(): return self._exchange.get_tickers() def refresh_pairlist(self) -> None: - """ - Run pairlist through all configured Pairlist Handlers. - """ + """Run pairlist through all configured Pairlist Handlers.""" # Tickers should be cached to avoid calling the exchange on each call. tickers: Dict = {} if self._tickers_needed: From 512e1633556ce9cdb4b8afed6205f36fb33feb7b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Dec 2020 09:48:49 +0100 Subject: [PATCH 035/183] change docstring to better reflect what the method is for --- docs/strategy-advanced.md | 2 +- freqtrade/strategy/interface.py | 2 +- freqtrade/templates/subtemplates/strategy_methods_advanced.j2 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 49720d729..519a9ac62 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -37,7 +37,7 @@ To simulate a regular trailing stoploss of 4% (trailing 4% behind the maximum re When not implemented by a strategy, returns the initial stoploss value Only called when use_custom_stoploss is set to True. - :param pair: Pair that's about to be sold. + :param pair: Pair that's currently analyzed :param trade: trade object. :param current_time: datetime object, containing the current datetime :param current_rate: Rate, calculated based on pricing settings in ask_strategy. diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 61c879e8f..1e4fc8b12 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -267,7 +267,7 @@ class IStrategy(ABC): When not implemented by a strategy, returns the initial stoploss value Only called when use_custom_stoploss is set to True. - :param pair: Pair that's about to be sold. + :param pair: Pair that's currently analyzed :param trade: trade object. :param current_time: datetime object, containing the current datetime :param current_rate: Rate, calculated based on pricing settings in ask_strategy. diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index f4cda2e89..53ededa19 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -69,7 +69,7 @@ def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: When not implemented by a strategy, returns True (always confirming). - :param pair: Pair that's about to be sold. + :param pair: Pair that's currently analyzed :param trade: trade object. :param order_type: Order type (as configured in order_types). usually limit or market. :param amount: Amount in quote currency. From e5840abaf9f77ecb47df29ca352f07ec0e4ae032 Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Thu, 31 Dec 2020 21:05:47 +0100 Subject: [PATCH 036/183] Added imports to documentation for clarification when using custom stoploss Signed-off-by: hoeckxer --- docs/strategy-advanced.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 519a9ac62..7876106be 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -23,6 +23,10 @@ E.g. `current_profit = 0.05` (5% profit) - stoploss returns `0.02` - then you "l To simulate a regular trailing stoploss of 4% (trailing 4% behind the maximum reached price) you would use the following very simple method: ``` python +# additional imports required +from freqtrade.persistence import Trade +from datetime import datetime + use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, From 67ced6a53c78a291c31b62e551cb961449b6ea83 Mon Sep 17 00:00:00 2001 From: Erwin Hoeckx Date: Fri, 1 Jan 2021 20:49:04 +0100 Subject: [PATCH 037/183] Update docs/strategy-advanced.md Co-authored-by: Matthias --- docs/strategy-advanced.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 7876106be..ab94be1c4 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -29,7 +29,7 @@ from datetime import datetime use_custom_stoploss = True - def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, current_profit: float, **kwargs) -> float: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). From 11f36fbaee6d72580cc649635f71ee635c45af8c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Jan 2021 09:14:31 +0100 Subject: [PATCH 038/183] Fix all custom stoploss samples --- docs/strategy-advanced.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index ab94be1c4..b1fcb50fc 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -24,8 +24,12 @@ To simulate a regular trailing stoploss of 4% (trailing 4% behind the maximum re ``` python # additional imports required -from freqtrade.persistence import Trade from datetime import datetime +from freqtrade.persistence import Trade + +class AwesomeStrategy(IStrategy): + + # ... populate_* methods use_custom_stoploss = True @@ -70,6 +74,13 @@ Of course, many more things are possible, and all examples can be combined at wi Use the initial stoploss for the first 60 minutes, after this change to 10% trailing stoploss, and after 2 hours (120 minutes) we use a 5% trailing stoploss. ``` python +from datetime import datetime, timedelta +from freqtrade.persistence import Trade + +class AwesomeStrategy(IStrategy): + + # ... populate_* methods + use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, @@ -89,6 +100,13 @@ Use a different stoploss depending on the pair. In this example, we'll trail the highest price with 10% trailing stoploss for `ETH/BTC` and `XRP/BTC`, with 5% trailing stoploss for `LTC/BTC` and with 15% for all other pairs. ``` python +from datetime import datetime +from freqtrade.persistence import Trade + +class AwesomeStrategy(IStrategy): + + # ... populate_* methods + use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, @@ -111,6 +129,13 @@ The below example sets absolute profit levels based on the current profit. * Once profit is > 20% - stoploss will be set to 7%. ``` python +from datetime import datetime +from freqtrade.persistence import Trade + +class AwesomeStrategy(IStrategy): + + # ... populate_* methods + use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, From 1e38fec61b42c998111297a56b8e2c7b2a304919 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 24 Dec 2020 06:55:19 +0100 Subject: [PATCH 039/183] Initial fastapi implementation (Ping working) --- freqtrade/rpc/api_server2/__init__.py | 2 + freqtrade/rpc/api_server2/api_v1.py | 14 +++++ freqtrade/rpc/api_server2/uvicorn_threaded.py | 28 ++++++++++ freqtrade/rpc/api_server2/webserver.py | 56 +++++++++++++++++++ freqtrade/rpc/rpc_manager.py | 5 +- requirements.txt | 4 ++ 6 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 freqtrade/rpc/api_server2/__init__.py create mode 100644 freqtrade/rpc/api_server2/api_v1.py create mode 100644 freqtrade/rpc/api_server2/uvicorn_threaded.py create mode 100644 freqtrade/rpc/api_server2/webserver.py diff --git a/freqtrade/rpc/api_server2/__init__.py b/freqtrade/rpc/api_server2/__init__.py new file mode 100644 index 000000000..df255c186 --- /dev/null +++ b/freqtrade/rpc/api_server2/__init__.py @@ -0,0 +1,2 @@ +# flake8: noqa: F401 +from .webserver import ApiServer diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server2/api_v1.py new file mode 100644 index 000000000..1e8bae1d4 --- /dev/null +++ b/freqtrade/rpc/api_server2/api_v1.py @@ -0,0 +1,14 @@ +from typing import Dict + +from fastapi import APIRouter + + +router = APIRouter() + + +@router.get('/ping') +def _ping() -> Dict[str, str]: + """simple ping version""" + return {"status": "pong"} + + diff --git a/freqtrade/rpc/api_server2/uvicorn_threaded.py b/freqtrade/rpc/api_server2/uvicorn_threaded.py new file mode 100644 index 000000000..ba9263620 --- /dev/null +++ b/freqtrade/rpc/api_server2/uvicorn_threaded.py @@ -0,0 +1,28 @@ +import contextlib +import time +import threading +import uvicorn + + +class UvicornServer(uvicorn.Server): + """ + Multithreaded server - as found in https://github.com/encode/uvicorn/issues/742 + """ + def install_signal_handlers(self): + pass + + @contextlib.contextmanager + def run_in_thread(self): + self.thread = threading.Thread(target=self.run) + self.thread.start() + # try: + while not self.started: + time.sleep(1e-3) + # yield + # finally: + # self.should_exit = True + # thread.join() + + def cleanup(self): + self.should_exit = True + self.thread.join() diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server2/webserver.py new file mode 100644 index 000000000..3f9baeaff --- /dev/null +++ b/freqtrade/rpc/api_server2/webserver.py @@ -0,0 +1,56 @@ +import threading +from typing import Any, Dict + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +import uvicorn + +from freqtrade.rpc.fiat_convert import CryptoToFiatConverter +from freqtrade.rpc.rpc import RPCHandler, RPC + +from .uvicorn_threaded import UvicornServer + + +class ApiServer(RPCHandler): + + def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: + super().__init__(rpc, config) + self._server = None + + self.app = FastAPI() + self.configure_app(self.app, self._config) + + self.start_api() + + def cleanup(self) -> None: + """ Cleanup pending module resources """ + if self._server: + self._server.cleanup() + + def send_msg(self, msg: Dict[str, str]) -> None: + pass + + def configure_app(self, app, config): + from .api_v1 import router as api_v1 + app.include_router(api_v1, prefix="/api/v1") + + app.add_middleware( + CORSMiddleware, + allow_origins=config['api_server'].get('CORS_origins', []), + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + def start_api(self): + """ + Start API ... should be run in thread. + """ + uvconfig = uvicorn.Config(self.app, + port=self._config['api_server'].get('listen_port', 8080), + host=self._config['api_server'].get( + 'listen_ip_address', '127.0.0.1'), + access_log=True) + self._server = UvicornServer(uvconfig) + + self._server.run_in_thread() diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 38a4e95fd..2afd39eda 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -34,7 +34,10 @@ class RPCManager: # Enable local rest api server for cmd line control if config.get('api_server', {}).get('enabled', False): logger.info('Enabling rpc.api_server') - from freqtrade.rpc.api_server import ApiServer + # from freqtrade.rpc.api_server import ApiServer + # TODO: Remove the above import + from freqtrade.rpc.api_server2 import ApiServer + self.registered_modules.append(ApiServer(self._rpc, config)) def cleanup(self) -> None: diff --git a/requirements.txt b/requirements.txt index bab13ed03..ad43c1006 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,6 +32,10 @@ flask==1.1.2 flask-jwt-extended==3.25.0 flask-cors==3.0.9 +# API Server +fastapi==0.63.0 +uvicorn==0.13.2 + # Support for colorized terminal output colorama==0.4.4 # Building config files interactively From a862f19f8282e9ee42eebaa0917dcafa320ba08e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 25 Dec 2020 13:08:25 +0100 Subject: [PATCH 040/183] Allow retrieval of rpc and config via dependencies --- freqtrade/rpc/api_server2/api_v1.py | 13 +++++++++--- freqtrade/rpc/api_server2/deps.py | 9 ++++++++ freqtrade/rpc/api_server2/models.py | 29 ++++++++++++++++++++++++++ freqtrade/rpc/api_server2/webserver.py | 14 ++++++++++--- 4 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 freqtrade/rpc/api_server2/deps.py create mode 100644 freqtrade/rpc/api_server2/models.py diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server2/api_v1.py index 1e8bae1d4..2d1491b06 100644 --- a/freqtrade/rpc/api_server2/api_v1.py +++ b/freqtrade/rpc/api_server2/api_v1.py @@ -1,14 +1,21 @@ from typing import Dict -from fastapi import APIRouter +from fastapi import APIRouter, Depends +from .deps import get_rpc, get_config +from .models import Balances, Ping +# Public API, requires no auth. +router_public = APIRouter() router = APIRouter() -@router.get('/ping') -def _ping() -> Dict[str, str]: +@router_public.get('/ping', response_model=Ping) +def ping(): """simple ping version""" return {"status": "pong"} +@router.get('/balance', response_model=Balances) +def balance(rpc=Depends(get_rpc), config=Depends(get_config)) -> Dict[str, str]: + return rpc._rpc_balance(config['stake_currency'], config.get('fiat_display_currency', ''),) diff --git a/freqtrade/rpc/api_server2/deps.py b/freqtrade/rpc/api_server2/deps.py new file mode 100644 index 000000000..60cc6b8fb --- /dev/null +++ b/freqtrade/rpc/api_server2/deps.py @@ -0,0 +1,9 @@ +from .webserver import ApiServer + + +def get_rpc(): + return ApiServer._rpc + + +def get_config(): + return ApiServer._config diff --git a/freqtrade/rpc/api_server2/models.py b/freqtrade/rpc/api_server2/models.py new file mode 100644 index 000000000..3d1fbf969 --- /dev/null +++ b/freqtrade/rpc/api_server2/models.py @@ -0,0 +1,29 @@ +from typing import List +from pydantic import BaseModel + + +class Ping(BaseModel): + status: str + + class Config: + schema_extra = { + "example": {"status", "pong"} + } + + +class Balance(BaseModel): + currency: str + free: float + balance: float + used: float + est_stake: float + stake: str + + +class Balances(BaseModel): + currencies: List[Balance] + total: float + symbol: str + value: float + stake: str + note: str diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server2/webserver.py index 3f9baeaff..23cae8b73 100644 --- a/freqtrade/rpc/api_server2/webserver.py +++ b/freqtrade/rpc/api_server2/webserver.py @@ -1,11 +1,9 @@ -import threading from typing import Any, Dict from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware import uvicorn -from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.rpc.rpc import RPCHandler, RPC from .uvicorn_threaded import UvicornServer @@ -13,10 +11,16 @@ from .uvicorn_threaded import UvicornServer class ApiServer(RPCHandler): + _rpc = None + _config = None + def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: super().__init__(rpc, config) self._server = None + ApiServer._rpc = rpc + ApiServer._config = config + self.app = FastAPI() self.configure_app(self.app, self._config) @@ -30,8 +34,12 @@ class ApiServer(RPCHandler): def send_msg(self, msg: Dict[str, str]) -> None: pass - def configure_app(self, app, config): + def configure_app(self, app: FastAPI, config): + from .api_v1 import router_public as api_v1_public from .api_v1 import router as api_v1 + app.include_router(api_v1_public, prefix="/api/v1") + + # TODO: Include auth dependency! app.include_router(api_v1, prefix="/api/v1") app.add_middleware( From 619b855d5fb203b31b9d4c731b988169f7aef256 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 25 Dec 2020 13:11:01 +0100 Subject: [PATCH 041/183] Add version endpoint --- freqtrade/rpc/api_server2/api_v1.py | 17 ++++++++++++----- freqtrade/rpc/api_server2/models.py | 7 +++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server2/api_v1.py index 2d1491b06..767d72023 100644 --- a/freqtrade/rpc/api_server2/api_v1.py +++ b/freqtrade/rpc/api_server2/api_v1.py @@ -1,12 +1,14 @@ -from typing import Dict - from fastapi import APIRouter, Depends -from .deps import get_rpc, get_config -from .models import Balances, Ping +from freqtrade import __version__ + +from .deps import get_config, get_rpc +from .models import Balances, Ping, Version + # Public API, requires no auth. router_public = APIRouter() +# Private API, protected by authentication router = APIRouter() @@ -17,5 +19,10 @@ def ping(): @router.get('/balance', response_model=Balances) -def balance(rpc=Depends(get_rpc), config=Depends(get_config)) -> Dict[str, str]: +def balance(rpc=Depends(get_rpc), config=Depends(get_config)): return rpc._rpc_balance(config['stake_currency'], config.get('fiat_display_currency', ''),) + + +@router.get('/version', response_model=Version) +def version(): + return {"version": __version__} diff --git a/freqtrade/rpc/api_server2/models.py b/freqtrade/rpc/api_server2/models.py index 3d1fbf969..7cd628da0 100644 --- a/freqtrade/rpc/api_server2/models.py +++ b/freqtrade/rpc/api_server2/models.py @@ -5,10 +5,9 @@ from pydantic import BaseModel class Ping(BaseModel): status: str - class Config: - schema_extra = { - "example": {"status", "pong"} - } + +class Version(BaseModel): + version: str class Balance(BaseModel): From eac74a9dece972a6fc6eae4d968a701ae6fbb843 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 25 Dec 2020 15:50:19 +0100 Subject: [PATCH 042/183] Implement auth in fastapi --- freqtrade/rpc/api_server2/auth.py | 83 ++++++++++++++++++++++++++ freqtrade/rpc/api_server2/models.py | 8 +++ freqtrade/rpc/api_server2/webserver.py | 24 ++++---- requirements.txt | 1 + 4 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 freqtrade/rpc/api_server2/auth.py diff --git a/freqtrade/rpc/api_server2/auth.py b/freqtrade/rpc/api_server2/auth.py new file mode 100644 index 000000000..3155e7754 --- /dev/null +++ b/freqtrade/rpc/api_server2/auth.py @@ -0,0 +1,83 @@ +from freqtrade.rpc.api_server2.models import AccessAndRefreshToken, AccessToken +import secrets +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from fastapi.security.http import HTTPBasic, HTTPBasicCredentials +from fastapi.security.utils import get_authorization_scheme_param +from fastapi_jwt_auth import AuthJWT +from pydantic import BaseModel + +from .deps import get_config + + +SECRET_KEY = "09d25e094faa6ca2556c818166b7a9563b93f7099f6f0f4caa6cf63b88e8d3e7" +ALGORITHM = "HS256" +ACCESS_TOKEN_EXPIRE_MINUTES = 30 + +router_login = APIRouter() + + +class Settings(BaseModel): + # TODO: should be set as config['api_server'].get('jwt_secret_key', 'super-secret') + authjwt_secret_key: str = "secret" + + +@AuthJWT.load_config +def get_jwt_config(): + return Settings() + + +def verify_auth(config, username: str, password: str): + return (secrets.compare_digest(username, config['api_server'].get('username')) and + secrets.compare_digest(password, config['api_server'].get('password'))) + + +class HTTPBasicOrJWTToken(HTTPBasic): + description = "Token Or Pass auth" + + async def __call__(self, request: Request, config=Depends(get_config) # type: ignore + ) -> Optional[str]: + header_authorization: str = request.headers.get("Authorization") + header_scheme, header_param = get_authorization_scheme_param(header_authorization) + if header_scheme.lower() == 'bearer': + AuthJWT.jwt_required() + elif header_scheme.lower() == 'basic': + credentials: Optional[HTTPBasicCredentials] = await HTTPBasic()(request) + if credentials and verify_auth(config, credentials.username, credentials.password): + return credentials.username + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect username or password", + headers={"WWW-Authenticate": "Basic"}, + ) + + +@router_login.post('/token/login', response_model=AccessAndRefreshToken) +def token_login(form_data: HTTPBasicCredentials = Depends(HTTPBasic()), config=Depends(get_config)): + + print(form_data) + Authorize = AuthJWT() + + if verify_auth(config, form_data.username, form_data.password): + token_data = form_data.username + access_token = Authorize.create_access_token(subject=token_data) + refresh_token = Authorize.create_refresh_token(subject=token_data) + return { + "access_token": access_token, + "refresh_token": refresh_token, + } + else: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect username or password", + headers={"WWW-Authenticate": "Basic"}, + ) + + +@router_login.post('/token/refresh', response_model=AccessToken) +def token_refresh(Authorize: AuthJWT = Depends()): + Authorize.jwt_refresh_token_required() + + access_token = Authorize.create_access_token(subject=Authorize.get_jwt_subject()) + return {'access_token': access_token} diff --git a/freqtrade/rpc/api_server2/models.py b/freqtrade/rpc/api_server2/models.py index 7cd628da0..c9ddb0d6f 100644 --- a/freqtrade/rpc/api_server2/models.py +++ b/freqtrade/rpc/api_server2/models.py @@ -6,6 +6,14 @@ class Ping(BaseModel): status: str +class AccessToken(BaseModel): + access_token: str + + +class AccessAndRefreshToken(AccessToken): + refresh_token: str + + class Version(BaseModel): version: str diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server2/webserver.py index 23cae8b73..84f6fc222 100644 --- a/freqtrade/rpc/api_server2/webserver.py +++ b/freqtrade/rpc/api_server2/webserver.py @@ -1,18 +1,17 @@ -from typing import Any, Dict - -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware +from typing import Any, Dict, Optional import uvicorn +from fastapi import Depends, FastAPI +from fastapi.middleware.cors import CORSMiddleware -from freqtrade.rpc.rpc import RPCHandler, RPC +from freqtrade.rpc.rpc import RPC, RPCHandler from .uvicorn_threaded import UvicornServer class ApiServer(RPCHandler): - _rpc = None - _config = None + _rpc: Optional[RPC] = None + _config: Dict[str, Any] = {} def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: super().__init__(rpc, config) @@ -21,7 +20,7 @@ class ApiServer(RPCHandler): ApiServer._rpc = rpc ApiServer._config = config - self.app = FastAPI() + self.app = FastAPI(title="Freqtrade API") self.configure_app(self.app, self._config) self.start_api() @@ -35,12 +34,15 @@ class ApiServer(RPCHandler): pass def configure_app(self, app: FastAPI, config): - from .api_v1 import router_public as api_v1_public from .api_v1 import router as api_v1 + from .api_v1 import router_public as api_v1_public + from .auth import router_login, HTTPBasicOrJWTToken app.include_router(api_v1_public, prefix="/api/v1") - # TODO: Include auth dependency! - app.include_router(api_v1, prefix="/api/v1") + app.include_router(api_v1, prefix="/api/v1", + dependencies=[Depends(HTTPBasicOrJWTToken())] + ) + app.include_router(router_login, prefix="/api/v1") app.add_middleware( CORSMiddleware, diff --git a/requirements.txt b/requirements.txt index ad43c1006..7cda9e48c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,6 +35,7 @@ flask-cors==3.0.9 # API Server fastapi==0.63.0 uvicorn==0.13.2 +fastapi_jwt_auth==0.5.0 # Support for colorized terminal output colorama==0.4.4 From 6594278509feb76b878fa07cbb77d52d5cb870a1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 25 Dec 2020 15:57:05 +0100 Subject: [PATCH 043/183] Reorder endpoints --- freqtrade/rpc/api_server2/api_v1.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server2/api_v1.py index 767d72023..c2e7a9172 100644 --- a/freqtrade/rpc/api_server2/api_v1.py +++ b/freqtrade/rpc/api_server2/api_v1.py @@ -18,11 +18,13 @@ def ping(): return {"status": "pong"} +@router.get('/version', response_model=Version) +def version(): + return {"version": __version__} + + @router.get('/balance', response_model=Balances) def balance(rpc=Depends(get_rpc), config=Depends(get_config)): return rpc._rpc_balance(config['stake_currency'], config.get('fiat_display_currency', ''),) -@router.get('/version', response_model=Version) -def version(): - return {"version": __version__} From 86d07008848086607aaa5183127760a069d6091b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 25 Dec 2020 20:07:12 +0100 Subject: [PATCH 044/183] Move models to apimodels --- .../rpc/api_server2/{auth.py => api_auth.py} | 6 +++--- .../api_server2/{models.py => api_models.py} | 5 +++++ freqtrade/rpc/api_server2/api_v1.py | 17 +++++++++++++---- freqtrade/rpc/api_server2/uvicorn_threaded.py | 3 ++- freqtrade/rpc/api_server2/webserver.py | 3 ++- 5 files changed, 25 insertions(+), 9 deletions(-) rename freqtrade/rpc/api_server2/{auth.py => api_auth.py} (94%) rename freqtrade/rpc/api_server2/{models.py => api_models.py} (91%) diff --git a/freqtrade/rpc/api_server2/auth.py b/freqtrade/rpc/api_server2/api_auth.py similarity index 94% rename from freqtrade/rpc/api_server2/auth.py rename to freqtrade/rpc/api_server2/api_auth.py index 3155e7754..d0c975480 100644 --- a/freqtrade/rpc/api_server2/auth.py +++ b/freqtrade/rpc/api_server2/api_auth.py @@ -1,4 +1,3 @@ -from freqtrade.rpc.api_server2.models import AccessAndRefreshToken, AccessToken import secrets from typing import Optional @@ -8,6 +7,8 @@ from fastapi.security.utils import get_authorization_scheme_param from fastapi_jwt_auth import AuthJWT from pydantic import BaseModel +from freqtrade.rpc.api_server2.api_models import AccessAndRefreshToken, AccessToken + from .deps import get_config @@ -41,7 +42,7 @@ class HTTPBasicOrJWTToken(HTTPBasic): header_authorization: str = request.headers.get("Authorization") header_scheme, header_param = get_authorization_scheme_param(header_authorization) if header_scheme.lower() == 'bearer': - AuthJWT.jwt_required() + AuthJWT(request).jwt_required() elif header_scheme.lower() == 'basic': credentials: Optional[HTTPBasicCredentials] = await HTTPBasic()(request) if credentials and verify_auth(config, credentials.username, credentials.password): @@ -49,7 +50,6 @@ class HTTPBasicOrJWTToken(HTTPBasic): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Incorrect username or password", - headers={"WWW-Authenticate": "Basic"}, ) diff --git a/freqtrade/rpc/api_server2/models.py b/freqtrade/rpc/api_server2/api_models.py similarity index 91% rename from freqtrade/rpc/api_server2/models.py rename to freqtrade/rpc/api_server2/api_models.py index c9ddb0d6f..0646ce3e6 100644 --- a/freqtrade/rpc/api_server2/models.py +++ b/freqtrade/rpc/api_server2/api_models.py @@ -1,4 +1,5 @@ from typing import List + from pydantic import BaseModel @@ -18,6 +19,10 @@ class Version(BaseModel): version: str +class StatusMsg(BaseModel): + status: str + + class Balance(BaseModel): currency: str free: float diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server2/api_v1.py index c2e7a9172..b3e0a7a4f 100644 --- a/freqtrade/rpc/api_server2/api_v1.py +++ b/freqtrade/rpc/api_server2/api_v1.py @@ -1,9 +1,10 @@ +from freqtrade.rpc import RPC from fastapi import APIRouter, Depends from freqtrade import __version__ +from .api_models import Balances, Ping, StatusMsg, Version from .deps import get_config, get_rpc -from .models import Balances, Ping, Version # Public API, requires no auth. @@ -18,13 +19,21 @@ def ping(): return {"status": "pong"} -@router.get('/version', response_model=Version) +@router.get('/version', response_model=Version, tags=['info']) def version(): return {"version": __version__} -@router.get('/balance', response_model=Balances) -def balance(rpc=Depends(get_rpc), config=Depends(get_config)): +@router.get('/balance', response_model=Balances, tags=['info']) +def balance(rpc: RPC = Depends(get_rpc), config=Depends(get_config)): return rpc._rpc_balance(config['stake_currency'], config.get('fiat_display_currency', ''),) +@router.post('/start', response_model=StatusMsg, tags=['botcontrol']) +def start(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_start() + + +@router.post('/stop', response_model=StatusMsg, tags=['botcontrol']) +def stop(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_stop() diff --git a/freqtrade/rpc/api_server2/uvicorn_threaded.py b/freqtrade/rpc/api_server2/uvicorn_threaded.py index ba9263620..7c8804fd3 100644 --- a/freqtrade/rpc/api_server2/uvicorn_threaded.py +++ b/freqtrade/rpc/api_server2/uvicorn_threaded.py @@ -1,6 +1,7 @@ import contextlib -import time import threading +import time + import uvicorn diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server2/webserver.py index 84f6fc222..5f2e1d6fe 100644 --- a/freqtrade/rpc/api_server2/webserver.py +++ b/freqtrade/rpc/api_server2/webserver.py @@ -1,4 +1,5 @@ from typing import Any, Dict, Optional + import uvicorn from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -36,7 +37,7 @@ class ApiServer(RPCHandler): def configure_app(self, app: FastAPI, config): from .api_v1 import router as api_v1 from .api_v1 import router_public as api_v1_public - from .auth import router_login, HTTPBasicOrJWTToken + from .api_auth import HTTPBasicOrJWTToken, router_login app.include_router(api_v1_public, prefix="/api/v1") app.include_router(api_v1, prefix="/api/v1", From 5e4c4cae06e27f0ad110d73f89e8f3fde4e4338a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Dec 2020 08:48:15 +0100 Subject: [PATCH 045/183] Fix auth providers --- freqtrade/rpc/api_server2/api_auth.py | 102 +++++++++++++++---------- freqtrade/rpc/api_server2/webserver.py | 6 +- requirements.txt | 2 +- 3 files changed, 66 insertions(+), 44 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_auth.py b/freqtrade/rpc/api_server2/api_auth.py index d0c975480..cf0168576 100644 --- a/freqtrade/rpc/api_server2/api_auth.py +++ b/freqtrade/rpc/api_server2/api_auth.py @@ -1,12 +1,11 @@ +from datetime import datetime, timedelta import secrets -from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Request, status from fastapi.security.http import HTTPBasic, HTTPBasicCredentials -from fastapi.security.utils import get_authorization_scheme_param -from fastapi_jwt_auth import AuthJWT from pydantic import BaseModel - +import jwt +from fastapi.security import OAuth2PasswordBearer from freqtrade.rpc.api_server2.api_models import AccessAndRefreshToken, AccessToken from .deps import get_config @@ -19,50 +18,72 @@ ACCESS_TOKEN_EXPIRE_MINUTES = 30 router_login = APIRouter() -class Settings(BaseModel): - # TODO: should be set as config['api_server'].get('jwt_secret_key', 'super-secret') - authjwt_secret_key: str = "secret" - - -@AuthJWT.load_config -def get_jwt_config(): - return Settings() - - def verify_auth(config, username: str, password: str): + """Verify username/password""" return (secrets.compare_digest(username, config['api_server'].get('username')) and secrets.compare_digest(password, config['api_server'].get('password'))) -class HTTPBasicOrJWTToken(HTTPBasic): - description = "Token Or Pass auth" +httpbasic = HTTPBasic(auto_error=False) +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token", auto_error=False) - async def __call__(self, request: Request, config=Depends(get_config) # type: ignore - ) -> Optional[str]: - header_authorization: str = request.headers.get("Authorization") - header_scheme, header_param = get_authorization_scheme_param(header_authorization) - if header_scheme.lower() == 'bearer': - AuthJWT(request).jwt_required() - elif header_scheme.lower() == 'basic': - credentials: Optional[HTTPBasicCredentials] = await HTTPBasic()(request) - if credentials and verify_auth(config, credentials.username, credentials.password): - return credentials.username - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Incorrect username or password", - ) + +def get_user_from_token(token, token_type: str = "access"): + credentials_exception = HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Could not validate credentials", + headers={"WWW-Authenticate": "Bearer"}, + ) + try: + payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) + username: str = payload.get("sub") + if username is None: + raise credentials_exception + if payload.get("type") != token_type: + raise credentials_exception + + except jwt.PyJWTError: + raise credentials_exception + return username + + +def create_token(data: dict, token_type: str = "access") -> str: + to_encode = data.copy() + if token_type == "access": + expire = datetime.utcnow() + timedelta(minutes=15) + elif token_type == "refresh": + expire = datetime.utcnow() + timedelta(days=30) + else: + raise ValueError() + to_encode.update({ + "exp": expire, + "iat": datetime.utcnow(), + "type": token_type, + }) + encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM) + return encoded_jwt + + +def http_basic_or_jwt_token(form_data: HTTPBasicCredentials = Depends(httpbasic), + token: str = Depends(oauth2_scheme), config=Depends(get_config)): + if token: + return get_user_from_token(token) + elif form_data and verify_auth(config, form_data.username, form_data.password): + return form_data.username + + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect username or password", + ) @router_login.post('/token/login', response_model=AccessAndRefreshToken) def token_login(form_data: HTTPBasicCredentials = Depends(HTTPBasic()), config=Depends(get_config)): - print(form_data) - Authorize = AuthJWT() - if verify_auth(config, form_data.username, form_data.password): - token_data = form_data.username - access_token = Authorize.create_access_token(subject=token_data) - refresh_token = Authorize.create_refresh_token(subject=token_data) + token_data = {'sub': form_data.username} + access_token = create_token(token_data) + refresh_token = create_token(token_data, token_type="refresh") return { "access_token": access_token, "refresh_token": refresh_token, @@ -76,8 +97,9 @@ def token_login(form_data: HTTPBasicCredentials = Depends(HTTPBasic()), config=D @router_login.post('/token/refresh', response_model=AccessToken) -def token_refresh(Authorize: AuthJWT = Depends()): - Authorize.jwt_refresh_token_required() - - access_token = Authorize.create_access_token(subject=Authorize.get_jwt_subject()) +def token_refresh(token: str = Depends(oauth2_scheme)): + # Refresh token + u = get_user_from_token(token, 'refresh') + token_data = {'sub': u} + access_token = create_token(token_data, token_type="access") return {'access_token': access_token} diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server2/webserver.py index 5f2e1d6fe..755b43127 100644 --- a/freqtrade/rpc/api_server2/webserver.py +++ b/freqtrade/rpc/api_server2/webserver.py @@ -37,13 +37,13 @@ class ApiServer(RPCHandler): def configure_app(self, app: FastAPI, config): from .api_v1 import router as api_v1 from .api_v1 import router_public as api_v1_public - from .api_auth import HTTPBasicOrJWTToken, router_login + from .api_auth import http_basic_or_jwt_token, router_login app.include_router(api_v1_public, prefix="/api/v1") app.include_router(api_v1, prefix="/api/v1", - dependencies=[Depends(HTTPBasicOrJWTToken())] + dependencies=[Depends(http_basic_or_jwt_token)], ) - app.include_router(router_login, prefix="/api/v1") + app.include_router(router_login, prefix="/api/v1", tags=["auth"]) app.add_middleware( CORSMiddleware, diff --git a/requirements.txt b/requirements.txt index 7cda9e48c..4b439079b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,7 +35,7 @@ flask-cors==3.0.9 # API Server fastapi==0.63.0 uvicorn==0.13.2 -fastapi_jwt_auth==0.5.0 +pyjwt==1.7.1 # Support for colorized terminal output colorama==0.4.4 From 4b86700a0fd0dac1ce95acf37ed59b831b7812b0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Dec 2020 15:54:22 +0100 Subject: [PATCH 046/183] Implement more endpoints --- freqtrade/rpc/api_server2/api_auth.py | 7 ++++--- freqtrade/rpc/api_server2/api_models.py | 6 ++++++ freqtrade/rpc/api_server2/api_v1.py | 24 ++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_auth.py b/freqtrade/rpc/api_server2/api_auth.py index cf0168576..cb19d7637 100644 --- a/freqtrade/rpc/api_server2/api_auth.py +++ b/freqtrade/rpc/api_server2/api_auth.py @@ -1,11 +1,12 @@ -from datetime import datetime, timedelta import secrets +from datetime import datetime, timedelta +import jwt from fastapi import APIRouter, Depends, HTTPException, Request, status +from fastapi.security import OAuth2PasswordBearer from fastapi.security.http import HTTPBasic, HTTPBasicCredentials from pydantic import BaseModel -import jwt -from fastapi.security import OAuth2PasswordBearer + from freqtrade.rpc.api_server2.api_models import AccessAndRefreshToken, AccessToken from .deps import get_config diff --git a/freqtrade/rpc/api_server2/api_models.py b/freqtrade/rpc/api_server2/api_models.py index 0646ce3e6..92e439b21 100644 --- a/freqtrade/rpc/api_server2/api_models.py +++ b/freqtrade/rpc/api_server2/api_models.py @@ -39,3 +39,9 @@ class Balances(BaseModel): value: float stake: str note: str + + +class Count(BaseModel): + current: int + max: int + total_stake: float diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server2/api_v1.py index b3e0a7a4f..e05db3ace 100644 --- a/freqtrade/rpc/api_server2/api_v1.py +++ b/freqtrade/rpc/api_server2/api_v1.py @@ -1,9 +1,9 @@ -from freqtrade.rpc import RPC from fastapi import APIRouter, Depends from freqtrade import __version__ +from freqtrade.rpc import RPC -from .api_models import Balances, Ping, StatusMsg, Version +from .api_models import Balances, Count, Ping, StatusMsg, Version from .deps import get_config, get_rpc @@ -29,6 +29,16 @@ def balance(rpc: RPC = Depends(get_rpc), config=Depends(get_config)): return rpc._rpc_balance(config['stake_currency'], config.get('fiat_display_currency', ''),) +@router.get('/count', response_model=Count, tags=['info']) +def count(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_count() + + +@router.get('/show_config', tags=['info']) +def show_config(rpc: RPC = Depends(get_rpc), config=Depends(get_config)): + return RPC._rpc_show_config(config, rpc._freqtrade.state) + + @router.post('/start', response_model=StatusMsg, tags=['botcontrol']) def start(rpc: RPC = Depends(get_rpc)): return rpc._rpc_start() @@ -37,3 +47,13 @@ def start(rpc: RPC = Depends(get_rpc)): @router.post('/stop', response_model=StatusMsg, tags=['botcontrol']) def stop(rpc: RPC = Depends(get_rpc)): return rpc._rpc_stop() + + +@router.post('/stopbuy', response_model=StatusMsg, tags=['botcontrol']) +def stop_buy(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_stopbuy() + + +@router.post('/reload_config', response_model=StatusMsg, tags=['botcontrol']) +def reload_config(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_reload_config() From f37ea4ba248923261648e2e509cc6e09f403365d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Dec 2020 16:33:13 +0100 Subject: [PATCH 047/183] Fix some initial tests towards fastAPI --- tests/rpc/test_rpc_apiserver.py | 76 +++++++++++++++++---------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 8da4ebfe7..7e777b732 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -7,6 +7,7 @@ from pathlib import Path from unittest.mock import ANY, MagicMock, PropertyMock import pytest +from fastapi.testclient import TestClient from flask import Flask from requests.auth import _basic_auth_str @@ -14,7 +15,8 @@ from freqtrade.__init__ import __version__ from freqtrade.loggers import setup_logging, setup_logging_pre from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC -from freqtrade.rpc.api_server import BASE_URI, ApiServer +from freqtrade.rpc.api_server import BASE_URI # , ApiServer +from freqtrade.rpc.api_server2 import ApiServer from freqtrade.state import RunMode, State from tests.conftest import create_mock_trades, get_patched_freqtradebot, log_has, patch_get_signal @@ -38,18 +40,19 @@ def botclient(default_conf, mocker): ftbot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(ftbot) - mocker.patch('freqtrade.rpc.api_server.ApiServer.run', MagicMock()) + mocker.patch('freqtrade.rpc.api_server2.ApiServer.start_api', MagicMock()) apiserver = ApiServer(rpc, default_conf) - yield ftbot, apiserver.app.test_client() + yield ftbot, TestClient(apiserver.app) # Cleanup ... ? def client_post(client, url, data={}): return client.post(url, - content_type="application/json", data=data, headers={'Authorization': _basic_auth_str(_TEST_USER, _TEST_PASS), - 'Origin': 'http://example.com'}) + 'Origin': 'http://example.com', + 'content-type': 'application/json' + }) def client_get(client, url): @@ -66,10 +69,10 @@ def client_delete(client, url): def assert_response(response, expected_code=200, needs_cors=True): assert response.status_code == expected_code - assert response.content_type == "application/json" + assert response.headers.get('content-type') == "application/json" if needs_cors: - assert ('Access-Control-Allow-Credentials', 'true') in response.headers._list - assert ('Access-Control-Allow-Origin', 'http://example.com') in response.headers._list + assert ('access-control-allow-credentials', 'true') in response.headers.items() + assert ('access-control-allow-origin', 'http://example.com') in response.headers.items() def test_api_not_found(botclient): @@ -77,7 +80,7 @@ def test_api_not_found(botclient): rc = client_post(client, f"{BASE_URI}/invalid_url") assert_response(rc, 404) - assert rc.json == {"status": "error", + assert rc.json() == {"status": "error", "reason": f"There's no API call for http://localhost{BASE_URI}/invalid_url.", "code": 404 } @@ -87,45 +90,44 @@ def test_api_unauthorized(botclient): ftbot, client = botclient rc = client.get(f"{BASE_URI}/ping") assert_response(rc, needs_cors=False) - assert rc.json == {'status': 'pong'} + assert rc.json() == {'status': 'pong'} # Don't send user/pass information rc = client.get(f"{BASE_URI}/version") assert_response(rc, 401, needs_cors=False) - assert rc.json == {'error': 'Unauthorized'} + assert rc.json() == {'error': 'Unauthorized'} # Change only username ftbot.config['api_server']['username'] = 'Ftrader' rc = client_get(client, f"{BASE_URI}/version") assert_response(rc, 401) - assert rc.json == {'error': 'Unauthorized'} + assert rc.json() == {'error': 'Unauthorized'} # Change only password ftbot.config['api_server']['username'] = _TEST_USER ftbot.config['api_server']['password'] = 'WrongPassword' rc = client_get(client, f"{BASE_URI}/version") assert_response(rc, 401) - assert rc.json == {'error': 'Unauthorized'} + assert rc.json() == {'error': 'Unauthorized'} ftbot.config['api_server']['username'] = 'Ftrader' ftbot.config['api_server']['password'] = 'WrongPassword' rc = client_get(client, f"{BASE_URI}/version") assert_response(rc, 401) - assert rc.json == {'error': 'Unauthorized'} + assert rc.json() == {'error': 'Unauthorized'} def test_api_token_login(botclient): ftbot, client = botclient rc = client_post(client, f"{BASE_URI}/token/login") assert_response(rc) - assert 'access_token' in rc.json - assert 'refresh_token' in rc.json + assert 'access_token' in rc.json() + assert 'refresh_token' in rc.json() # test Authentication is working with JWT tokens too rc = client.get(f"{BASE_URI}/count", - content_type="application/json", - headers={'Authorization': f'Bearer {rc.json["access_token"]}', + headers={'Authorization': f'Bearer {rc.json()["access_token"]}', 'Origin': 'http://example.com'}) assert_response(rc) @@ -268,7 +270,7 @@ def test_api_reloadconf(botclient): rc = client_post(client, f"{BASE_URI}/reload_config") assert_response(rc) - assert rc.json == {'status': 'Reloading config ...'} + assert rc.json() == {'status': 'Reloading config ...'} assert ftbot.state == State.RELOAD_CONFIG @@ -278,7 +280,7 @@ def test_api_stopbuy(botclient): rc = client_post(client, f"{BASE_URI}/stopbuy") assert_response(rc) - assert rc.json == {'status': 'No more buy will occur from now. Run /reload_config to reset.'} + assert rc.json() == {'status': 'No more buy will occur from now. Run /reload_config to reset.'} assert ftbot.config['max_open_trades'] == 0 @@ -293,9 +295,9 @@ def test_api_balance(botclient, mocker, rpc_balance): rc = client_get(client, f"{BASE_URI}/balance") assert_response(rc) - assert "currencies" in rc.json - assert len(rc.json["currencies"]) == 5 - assert rc.json['currencies'][0] == { + assert "currencies" in rc.json() + assert len(rc.json()["currencies"]) == 5 + assert rc.json()['currencies'][0] == { 'currency': 'BTC', 'free': 12.0, 'balance': 12.0, @@ -318,15 +320,15 @@ def test_api_count(botclient, mocker, ticker, fee, markets): rc = client_get(client, f"{BASE_URI}/count") assert_response(rc) - assert rc.json["current"] == 0 - assert rc.json["max"] == 1.0 + assert rc.json()["current"] == 0 + assert rc.json()["max"] == 1.0 # Create some test data ftbot.enter_positions() rc = client_get(client, f"{BASE_URI}/count") assert_response(rc) - assert rc.json["current"] == 1.0 - assert rc.json["max"] == 1.0 + assert rc.json()["current"] == 1.0 + assert rc.json()["max"] == 1.0 def test_api_locks(botclient): @@ -359,15 +361,15 @@ def test_api_show_config(botclient, mocker): rc = client_get(client, f"{BASE_URI}/show_config") assert_response(rc) - assert 'dry_run' in rc.json - assert rc.json['exchange'] == 'bittrex' - assert rc.json['timeframe'] == '5m' - assert rc.json['timeframe_ms'] == 300000 - assert rc.json['timeframe_min'] == 5 - assert rc.json['state'] == 'running' - assert not rc.json['trailing_stop'] - assert 'bid_strategy' in rc.json - assert 'ask_strategy' in rc.json + assert 'dry_run' in rc.json() + assert rc.json()['exchange'] == 'bittrex' + assert rc.json()['timeframe'] == '5m' + assert rc.json()['timeframe_ms'] == 300000 + assert rc.json()['timeframe_min'] == 5 + assert rc.json()['state'] == 'running' + assert not rc.json()['trailing_stop'] + assert 'bid_strategy' in rc.json() + assert 'ask_strategy' in rc.json() def test_api_daily(botclient, mocker, ticker, fee, markets): @@ -722,7 +724,7 @@ def test_api_version(botclient): rc = client_get(client, f"{BASE_URI}/version") assert_response(rc) - assert rc.json == {"version": __version__} + assert rc.json() == {"version": __version__} def test_api_blacklist(botclient, mocker): From a18d66e10814e4c0f53618a07c3c5f1a0df17c31 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Dec 2020 16:43:15 +0100 Subject: [PATCH 048/183] Add more endpoints to fastapi --- freqtrade/rpc/api_server2/api_models.py | 47 ++++++++++++++- freqtrade/rpc/api_server2/api_v1.py | 21 ++++++- tests/rpc/test_rpc_apiserver.py | 80 ++++++++++++------------- 3 files changed, 106 insertions(+), 42 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_models.py b/freqtrade/rpc/api_server2/api_models.py index 92e439b21..293b1e97f 100644 --- a/freqtrade/rpc/api_server2/api_models.py +++ b/freqtrade/rpc/api_server2/api_models.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Dict, List, Union from pydantic import BaseModel @@ -45,3 +45,48 @@ class Count(BaseModel): current: int max: int total_stake: float + + +class PerformanceEntry(BaseModel): + pair: str + profit: float + count: int + + +class Profit(BaseModel): + profit_closed_coin: float + profit_closed_percent: float + profit_closed_percent_mean: float + profit_closed_ratio_mean: float + profit_closed_percent_sum: float + profit_closed_ratio_sum: float + profit_closed_fiat: float + profit_all_coin: float + profit_all_percent: float + profit_all_percent_mean: float + profit_all_ratio_mean: float + profit_all_percent_sum: float + profit_all_ratio_sum: float + profit_all_fiat: float + trade_count: int + closed_trade_count: int + first_trade_date: str + first_trade_timestamp: int + latest_trade_date: str + latest_trade_timestamp: int + avg_duration: str + best_pair: str + best_rate: float + winning_trades: int + losing_trades: int + + +class SellReason(BaseModel): + wins: int + losses: int + draws: int + + +class Stats(BaseModel): + sell_reasons: Dict[str, SellReason] + durations: Dict[str, Union[str, float]] diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server2/api_v1.py index e05db3ace..78a7ebbf8 100644 --- a/freqtrade/rpc/api_server2/api_v1.py +++ b/freqtrade/rpc/api_server2/api_v1.py @@ -1,9 +1,11 @@ +from typing import List + from fastapi import APIRouter, Depends from freqtrade import __version__ from freqtrade.rpc import RPC -from .api_models import Balances, Count, Ping, StatusMsg, Version +from .api_models import Balances, Count, PerformanceEntry, Ping, Profit, Stats, StatusMsg, Version from .deps import get_config, get_rpc @@ -34,6 +36,23 @@ def count(rpc: RPC = Depends(get_rpc)): return rpc._rpc_count() +@router.get('/performance', response_model=List[PerformanceEntry], tags=['info']) +def performance(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_performance() + + +@router.get('/profit', response_model=Profit, tags=['info']) +def profit(rpc: RPC = Depends(get_rpc), config=Depends(get_config)): + return rpc._rpc_trade_statistics(config['stake_currency'], + config.get('fiat_display_currency') + ) + + +@router.get('/stats', response_model=Stats, tags=['info']) +def stats(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_stats() + + @router.get('/show_config', tags=['info']) def show_config(rpc: RPC = Depends(get_rpc), config=Depends(get_config)): return RPC._rpc_show_config(config, rpc._freqtrade.state) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 7e777b732..b700714a8 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -512,7 +512,7 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li rc = client_get(client, f"{BASE_URI}/profit") assert_response(rc, 200) - assert rc.json['trade_count'] == 0 + assert rc.json()['trade_count'] == 0 ftbot.enter_positions() trade = Trade.query.first() @@ -522,9 +522,9 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li rc = client_get(client, f"{BASE_URI}/profit") assert_response(rc, 200) # One open trade - assert rc.json['trade_count'] == 1 - assert rc.json['best_pair'] == '' - assert rc.json['best_rate'] == 0 + assert rc.json()['trade_count'] == 1 + assert rc.json()['best_pair'] == '' + assert rc.json()['best_rate'] == 0 trade = Trade.query.first() trade.update(limit_sell_order) @@ -534,32 +534,32 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li rc = client_get(client, f"{BASE_URI}/profit") assert_response(rc) - assert rc.json == {'avg_duration': '0:00:00', - 'best_pair': 'ETH/BTC', - 'best_rate': 6.2, - 'first_trade_date': 'just now', - 'first_trade_timestamp': ANY, - 'latest_trade_date': 'just now', - 'latest_trade_timestamp': ANY, - 'profit_all_coin': 6.217e-05, - 'profit_all_fiat': 0.76748865, - 'profit_all_percent': 6.2, - 'profit_all_percent_mean': 6.2, - 'profit_all_ratio_mean': 0.06201058, - 'profit_all_percent_sum': 6.2, - 'profit_all_ratio_sum': 0.06201058, - 'profit_closed_coin': 6.217e-05, - 'profit_closed_fiat': 0.76748865, - 'profit_closed_percent': 6.2, - 'profit_closed_ratio_mean': 0.06201058, - 'profit_closed_percent_mean': 6.2, - 'profit_closed_ratio_sum': 0.06201058, - 'profit_closed_percent_sum': 6.2, - 'trade_count': 1, - 'closed_trade_count': 1, - 'winning_trades': 1, - 'losing_trades': 0, - } + assert rc.json() == {'avg_duration': '0:00:00', + 'best_pair': 'ETH/BTC', + 'best_rate': 6.2, + 'first_trade_date': 'just now', + 'first_trade_timestamp': ANY, + 'latest_trade_date': 'just now', + 'latest_trade_timestamp': ANY, + 'profit_all_coin': 6.217e-05, + 'profit_all_fiat': 0.76748865, + 'profit_all_percent': 6.2, + 'profit_all_percent_mean': 6.2, + 'profit_all_ratio_mean': 0.06201058, + 'profit_all_percent_sum': 6.2, + 'profit_all_ratio_sum': 0.06201058, + 'profit_closed_coin': 6.217e-05, + 'profit_closed_fiat': 0.76748865, + 'profit_closed_percent': 6.2, + 'profit_closed_ratio_mean': 0.06201058, + 'profit_closed_percent_mean': 6.2, + 'profit_closed_ratio_sum': 0.06201058, + 'profit_closed_percent_sum': 6.2, + 'trade_count': 1, + 'closed_trade_count': 1, + 'winning_trades': 1, + 'losing_trades': 0, + } @pytest.mark.usefixtures("init_persistence") @@ -576,19 +576,19 @@ def test_api_stats(botclient, mocker, ticker, fee, markets,): rc = client_get(client, f"{BASE_URI}/stats") assert_response(rc, 200) - assert 'durations' in rc.json - assert 'sell_reasons' in rc.json + assert 'durations' in rc.json() + assert 'sell_reasons' in rc.json() create_mock_trades(fee) rc = client_get(client, f"{BASE_URI}/stats") assert_response(rc, 200) - assert 'durations' in rc.json - assert 'sell_reasons' in rc.json + assert 'durations' in rc.json() + assert 'sell_reasons' in rc.json() - assert 'wins' in rc.json['durations'] - assert 'losses' in rc.json['durations'] - assert 'draws' in rc.json['durations'] + assert 'wins' in rc.json()['durations'] + assert 'losses' in rc.json()['durations'] + assert 'draws' in rc.json()['durations'] def test_api_performance(botclient, mocker, ticker, fee): @@ -629,9 +629,9 @@ def test_api_performance(botclient, mocker, ticker, fee): rc = client_get(client, f"{BASE_URI}/performance") assert_response(rc) - assert len(rc.json) == 2 - assert rc.json == [{'count': 1, 'pair': 'LTC/ETH', 'profit': 7.61}, - {'count': 1, 'pair': 'XRP/ETH', 'profit': -5.57}] + assert len(rc.json()) == 2 + assert rc.json() == [{'count': 1, 'pair': 'LTC/ETH', 'profit': 7.61}, + {'count': 1, 'pair': 'XRP/ETH', 'profit': -5.57}] def test_api_status(botclient, mocker, ticker, fee, markets): From 73a29e6d7480857fb2ba2af81df8ac95c1bd0b45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Dec 2020 17:00:30 +0100 Subject: [PATCH 049/183] Improve tests, implement more fastapi methods --- freqtrade/rpc/api_server2/api_models.py | 11 +++++++++- freqtrade/rpc/api_server2/api_v1.py | 27 ++++++++++++++++++++++++- freqtrade/rpc/api_server2/webserver.py | 19 +++++++++++++++-- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_models.py b/freqtrade/rpc/api_server2/api_models.py index 293b1e97f..aa9dfcc33 100644 --- a/freqtrade/rpc/api_server2/api_models.py +++ b/freqtrade/rpc/api_server2/api_models.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Union +from typing import Dict, List, Optional, Union from pydantic import BaseModel @@ -90,3 +90,12 @@ class SellReason(BaseModel): class Stats(BaseModel): sell_reasons: Dict[str, SellReason] durations: Dict[str, Union[str, float]] + + +class ForceBuyPayload(BaseModel): + pair: str + price: Optional[float] + + +class ForceSellPayload(BaseModel): + tradeid: str diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server2/api_v1.py index 78a7ebbf8..961eb2c78 100644 --- a/freqtrade/rpc/api_server2/api_v1.py +++ b/freqtrade/rpc/api_server2/api_v1.py @@ -4,8 +4,10 @@ from fastapi import APIRouter, Depends from freqtrade import __version__ from freqtrade.rpc import RPC +from freqtrade.rpc.rpc import RPCException -from .api_models import Balances, Count, PerformanceEntry, Ping, Profit, Stats, StatusMsg, Version +from .api_models import (Balances, Count, ForceBuyPayload, ForceSellPayload, PerformanceEntry, Ping, Profit, Stats, + StatusMsg, Version) from .deps import get_config, get_rpc @@ -53,10 +55,33 @@ def stats(rpc: RPC = Depends(get_rpc)): return rpc._rpc_stats() +# TODO: Missing response model +@router.get('/status', tags=['info']) +def status(rpc: RPC = Depends(get_rpc)): + try: + return rpc._rpc_trade_status() + except RPCException: + return [] + + @router.get('/show_config', tags=['info']) def show_config(rpc: RPC = Depends(get_rpc), config=Depends(get_config)): return RPC._rpc_show_config(config, rpc._freqtrade.state) +@router.post('/forcebuy', tags=['trading']) +def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)): + trade = rpc._rpc_forcebuy(payload.pair, payload.price) + + if trade: + return trade.to_json() + else: + return {"status": f"Error buying pair {payload.pair}."} + + +@router.post('/forcesell', tags=['trading']) +def forcesell(payload: ForceSellPayload, rpc: RPC = Depends(get_rpc)): + return rpc._rpc_forcesell(payload.tradeid) + @router.post('/start', response_model=StatusMsg, tags=['botcontrol']) def start(rpc: RPC = Depends(get_rpc)): diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server2/webserver.py index 755b43127..45e695151 100644 --- a/freqtrade/rpc/api_server2/webserver.py +++ b/freqtrade/rpc/api_server2/webserver.py @@ -1,14 +1,20 @@ +import logging from typing import Any, Dict, Optional +from starlette.responses import JSONResponse import uvicorn from fastapi import Depends, FastAPI +from fastapi.exceptions import HTTPException from fastapi.middleware.cors import CORSMiddleware -from freqtrade.rpc.rpc import RPC, RPCHandler +from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler from .uvicorn_threaded import UvicornServer +logger = logging.getLogger(__name__) + + class ApiServer(RPCHandler): _rpc: Optional[RPC] = None @@ -34,10 +40,17 @@ class ApiServer(RPCHandler): def send_msg(self, msg: Dict[str, str]) -> None: pass + def handle_rpc_exception(self, request, exc): + logger.exception(f"API Error calling: {exc}") + return JSONResponse( + status_code=502, + content={'error': f"Error querying {request.url.path}: {exc.message}"} + ) + def configure_app(self, app: FastAPI, config): + from .api_auth import http_basic_or_jwt_token, router_login from .api_v1 import router as api_v1 from .api_v1 import router_public as api_v1_public - from .api_auth import http_basic_or_jwt_token, router_login app.include_router(api_v1_public, prefix="/api/v1") app.include_router(api_v1, prefix="/api/v1", @@ -53,6 +66,8 @@ class ApiServer(RPCHandler): allow_headers=["*"], ) + app.add_exception_handler(RPCException, self.handle_rpc_exception) + def start_api(self): """ Start API ... should be run in thread. From 9ee1d8835595cd688efd7e4db7eca4dd452d3249 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Dec 2020 17:33:27 +0100 Subject: [PATCH 050/183] Implement more endpoints --- freqtrade/rpc/api_server2/api_models.py | 65 ++++- freqtrade/rpc/api_server2/api_v1.py | 59 +++- tests/rpc/test_rpc_apiserver.py | 346 ++++++++++++------------ 3 files changed, 295 insertions(+), 175 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_models.py b/freqtrade/rpc/api_server2/api_models.py index aa9dfcc33..a30973845 100644 --- a/freqtrade/rpc/api_server2/api_models.py +++ b/freqtrade/rpc/api_server2/api_models.py @@ -1,5 +1,5 @@ from typing import Dict, List, Optional, Union - +from datetime import date from pydantic import BaseModel @@ -23,6 +23,10 @@ class StatusMsg(BaseModel): status: str +class ResultMsg(BaseModel): + result: str + + class Balance(BaseModel): currency: str free: float @@ -92,6 +96,39 @@ class Stats(BaseModel): durations: Dict[str, Union[str, float]] +class DailyRecord(BaseModel): + date: date + abs_profit: float + fiat_value: float + trade_count: int + + +class Daily(BaseModel): + data: List[DailyRecord] + fiat_display_currency: str + stake_currency: str + + +class LockModel(BaseModel): + active: bool + lock_end_time: str + lock_end_timestamp: int + lock_time: str + lock_timestamp: int + pair: str + reason: str + + +class Locks(BaseModel): + lock_count: int + locks: List[LockModel] + + +class Logs(BaseModel): + log_count: int + logs: List[List] + + class ForceBuyPayload(BaseModel): pair: str price: Optional[float] @@ -99,3 +136,29 @@ class ForceBuyPayload(BaseModel): class ForceSellPayload(BaseModel): tradeid: str + + +class BlacklistPayload(BaseModel): + blacklist: List[str] + + +class BlacklistResponse(BaseModel): + blacklist: List[str] + blacklist_expanded: List[str] + errors: Dict + length: int + method: List[str] + + +class WhitelistResponse(BaseModel): + whitelist: List[str] + length: int + method: List[str] + + + +class DeleteTrade(BaseModel): + cancel_order_count: int + result: str + result_msg: str + trade_id: int diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server2/api_v1.py index 961eb2c78..00964f162 100644 --- a/freqtrade/rpc/api_server2/api_v1.py +++ b/freqtrade/rpc/api_server2/api_v1.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from fastapi import APIRouter, Depends @@ -6,8 +6,8 @@ from freqtrade import __version__ from freqtrade.rpc import RPC from freqtrade.rpc.rpc import RPCException -from .api_models import (Balances, Count, ForceBuyPayload, ForceSellPayload, PerformanceEntry, Ping, Profit, Stats, - StatusMsg, Version) +from .api_models import (Balances, BlacklistPayload, BlacklistResponse, Count, Daily, DeleteTrade, ForceBuyPayload, ForceSellPayload, Locks, Logs, PerformanceEntry, Ping, Profit, ResultMsg, Stats, + StatusMsg, Version, WhitelistResponse) from .deps import get_config, get_rpc @@ -55,6 +55,12 @@ def stats(rpc: RPC = Depends(get_rpc)): return rpc._rpc_stats() +@router.get('/daily', response_model=Daily, tags=['info']) +def daily(timescale: int = 7, rpc: RPC = Depends(get_rpc), config=Depends(get_config)): + return rpc._rpc_daily_profit(timescale, config['stake_currency'], + config.get('fiat_display_currency', '')) + + # TODO: Missing response model @router.get('/status', tags=['info']) def status(rpc: RPC = Depends(get_rpc)): @@ -64,10 +70,30 @@ def status(rpc: RPC = Depends(get_rpc)): return [] +# TODO: Missing response model +@router.get('/trades', tags=['info']) +def trades(limit: Optional[int] = 0, rpc: RPC = Depends(get_rpc)): + return rpc._rpc_trade_history(limit) + + +@router.delete('/trades/{tradeid}', response_model=DeleteTrade, tags=['info', 'trading']) +def trades_delete(tradeid: int, rpc: RPC = Depends(get_rpc)): + return rpc._rpc_delete(tradeid) + + +# TODO: Missing response model +@router.get('/edge', tags=['info']) +def edge(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_edge() + + +# TODO: Missing response model @router.get('/show_config', tags=['info']) def show_config(rpc: RPC = Depends(get_rpc), config=Depends(get_config)): return RPC._rpc_show_config(config, rpc._freqtrade.state) + +# TODO: Missing response model @router.post('/forcebuy', tags=['trading']) def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)): trade = rpc._rpc_forcebuy(payload.pair, payload.price) @@ -78,11 +104,36 @@ def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)): return {"status": f"Error buying pair {payload.pair}."} -@router.post('/forcesell', tags=['trading']) +@router.post('/forcesell', response_model=ResultMsg, tags=['trading']) def forcesell(payload: ForceSellPayload, rpc: RPC = Depends(get_rpc)): return rpc._rpc_forcesell(payload.tradeid) +@router.get('/blacklist', response_model=BlacklistResponse, tags=['info', 'pairlist']) +def blacklist(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_blacklist() + + +@router.post('/blacklist', response_model=BlacklistResponse, tags=['info', 'pairlist']) +def blacklist_post(payload: BlacklistPayload, rpc: RPC = Depends(get_rpc)): + return rpc._rpc_blacklist(payload.blacklist) + + +@router.get('/whitelist', response_model=WhitelistResponse, tags=['info', 'pairlist']) +def whitelist(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_whitelist() + + +@router.get('/locks', response_model=Locks, tags=['info']) +def locks(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_locks() + + +@router.get('/logs', response_model=Logs, tags=['info']) +def logs(limit: Optional[int] = None, rpc: RPC = Depends(get_rpc)): + return rpc._rpc_get_logs(limit) + + @router.post('/start', response_model=StatusMsg, tags=['botcontrol']) def start(rpc: RPC = Depends(get_rpc)): return rpc._rpc_start() diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index b700714a8..49ca4aa9c 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -337,10 +337,10 @@ def test_api_locks(botclient): rc = client_get(client, f"{BASE_URI}/locks") assert_response(rc) - assert 'locks' in rc.json + assert 'locks' in rc.json() - assert rc.json['lock_count'] == 0 - assert rc.json['lock_count'] == len(rc.json['locks']) + assert rc.json()['lock_count'] == 0 + assert rc.json()['lock_count'] == len(rc.json()['locks']) PairLocks.lock_pair('ETH/BTC', datetime.now(timezone.utc) + timedelta(minutes=4), 'randreason') PairLocks.lock_pair('XRP/BTC', datetime.now(timezone.utc) + timedelta(minutes=20), 'deadbeef') @@ -348,11 +348,11 @@ def test_api_locks(botclient): rc = client_get(client, f"{BASE_URI}/locks") assert_response(rc) - assert rc.json['lock_count'] == 2 - assert rc.json['lock_count'] == len(rc.json['locks']) - assert 'ETH/BTC' in (rc.json['locks'][0]['pair'], rc.json['locks'][1]['pair']) - assert 'randreason' in (rc.json['locks'][0]['reason'], rc.json['locks'][1]['reason']) - assert 'deadbeef' in (rc.json['locks'][0]['reason'], rc.json['locks'][1]['reason']) + assert rc.json()['lock_count'] == 2 + assert rc.json()['lock_count'] == len(rc.json()['locks']) + assert 'ETH/BTC' in (rc.json()['locks'][0]['pair'], rc.json()['locks'][1]['pair']) + assert 'randreason' in (rc.json()['locks'][0]['reason'], rc.json()['locks'][1]['reason']) + assert 'deadbeef' in (rc.json()['locks'][0]['reason'], rc.json()['locks'][1]['reason']) def test_api_show_config(botclient, mocker): @@ -384,10 +384,10 @@ def test_api_daily(botclient, mocker, ticker, fee, markets): ) rc = client_get(client, f"{BASE_URI}/daily") assert_response(rc) - assert len(rc.json['data']) == 7 - assert rc.json['stake_currency'] == 'BTC' - assert rc.json['fiat_display_currency'] == 'USD' - assert rc.json['data'][0]['date'] == str(datetime.utcnow().date()) + assert len(rc.json()['data']) == 7 + assert rc.json()['stake_currency'] == 'BTC' + assert rc.json()['fiat_display_currency'] == 'USD' + assert rc.json()['data'][0]['date'] == str(datetime.utcnow().date()) def test_api_trades(botclient, mocker, fee, markets): @@ -399,19 +399,20 @@ def test_api_trades(botclient, mocker, fee, markets): ) rc = client_get(client, f"{BASE_URI}/trades") assert_response(rc) - assert len(rc.json) == 2 - assert rc.json['trades_count'] == 0 + assert len(rc.json()) == 2 + assert rc.json()['trades_count'] == 0 create_mock_trades(fee) + Trade.session.flush() rc = client_get(client, f"{BASE_URI}/trades") assert_response(rc) - assert len(rc.json['trades']) == 2 - assert rc.json['trades_count'] == 2 + assert len(rc.json()['trades']) == 2 + assert rc.json()['trades_count'] == 2 rc = client_get(client, f"{BASE_URI}/trades?limit=1") assert_response(rc) - assert len(rc.json['trades']) == 1 - assert rc.json['trades_count'] == 1 + assert len(rc.json()['trades']) == 1 + assert rc.json()['trades_count'] == 1 def test_api_delete_trade(botclient, mocker, fee, markets): @@ -430,6 +431,7 @@ def test_api_delete_trade(botclient, mocker, fee, markets): assert_response(rc, 502) create_mock_trades(fee) + Trade.session.flush() ftbot.strategy.order_types['stoploss_on_exchange'] = True trades = Trade.query.all() trades[1].stoploss_order_id = '1234' @@ -437,7 +439,7 @@ def test_api_delete_trade(botclient, mocker, fee, markets): rc = client_delete(client, f"{BASE_URI}/trades/1") assert_response(rc) - assert rc.json['result_msg'] == 'Deleted trade 1. Closed 1 open orders.' + assert rc.json()['result_msg'] == 'Deleted trade 1. Closed 1 open orders.' assert len(trades) - 1 == len(Trade.query.all()) assert cancel_mock.call_count == 1 @@ -450,7 +452,7 @@ def test_api_delete_trade(botclient, mocker, fee, markets): assert len(trades) - 1 == len(Trade.query.all()) rc = client_delete(client, f"{BASE_URI}/trades/2") assert_response(rc) - assert rc.json['result_msg'] == 'Deleted trade 2. Closed 2 open orders.' + assert rc.json()['result_msg'] == 'Deleted trade 2. Closed 2 open orders.' assert len(trades) - 2 == len(Trade.query.all()) assert stoploss_mock.call_count == 1 @@ -459,28 +461,28 @@ def test_api_logs(botclient): ftbot, client = botclient rc = client_get(client, f"{BASE_URI}/logs") assert_response(rc) - assert len(rc.json) == 2 - assert 'logs' in rc.json + assert len(rc.json()) == 2 + assert 'logs' in rc.json() # Using a fixed comparison here would make this test fail! - assert rc.json['log_count'] > 1 - assert len(rc.json['logs']) == rc.json['log_count'] + assert rc.json()['log_count'] > 1 + assert len(rc.json()['logs']) == rc.json()['log_count'] - assert isinstance(rc.json['logs'][0], list) + assert isinstance(rc.json()['logs'][0], list) # date - assert isinstance(rc.json['logs'][0][0], str) + assert isinstance(rc.json()['logs'][0][0], str) # created_timestamp - assert isinstance(rc.json['logs'][0][1], float) - assert isinstance(rc.json['logs'][0][2], str) - assert isinstance(rc.json['logs'][0][3], str) - assert isinstance(rc.json['logs'][0][4], str) + assert isinstance(rc.json()['logs'][0][1], float) + assert isinstance(rc.json()['logs'][0][2], str) + assert isinstance(rc.json()['logs'][0][3], str) + assert isinstance(rc.json()['logs'][0][4], str) rc = client_get(client, f"{BASE_URI}/logs?limit=5") assert_response(rc) - assert len(rc.json) == 2 - assert 'logs' in rc.json + assert len(rc.json()) == 2 + assert 'logs' in rc.json() # Using a fixed comparison here would make this test fail! - assert rc.json['log_count'] == 5 - assert len(rc.json['logs']) == rc.json['log_count'] + assert rc.json()['log_count'] == 5 + assert len(rc.json()['logs']) == rc.json()['log_count'] def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): @@ -495,7 +497,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): ) rc = client_get(client, f"{BASE_URI}/edge") assert_response(rc, 502) - assert rc.json == {"error": "Error querying _edge: Edge is not enabled."} + assert rc.json() == {"error": "Error querying /api/v1/edge: Edge is not enabled."} @pytest.mark.usefixtures("init_persistence") @@ -647,7 +649,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets): rc = client_get(client, f"{BASE_URI}/status") assert_response(rc, 200) - assert rc.json == [] + assert rc.json() == [] ftbot.enter_positions() trades = Trade.get_open_trades() @@ -656,67 +658,68 @@ def test_api_status(botclient, mocker, ticker, fee, markets): rc = client_get(client, f"{BASE_URI}/status") assert_response(rc) - assert len(rc.json) == 1 - assert rc.json == [{'amount': 91.07468123, - 'amount_requested': 91.07468123, - 'base_currency': 'BTC', - 'close_date': None, - 'close_date_hum': None, - 'close_timestamp': None, - 'close_profit': None, - 'close_profit_pct': None, - 'close_profit_abs': None, - 'close_rate': None, - 'current_profit': -0.00408133, - 'current_profit_pct': -0.41, - 'current_profit_abs': -4.09e-06, - 'profit_ratio': -0.00408133, - 'profit_pct': -0.41, - 'profit_abs': -4.09e-06, - 'current_rate': 1.099e-05, - 'open_date': ANY, - 'open_date_hum': 'just now', - 'open_timestamp': ANY, - 'open_order': None, - 'open_rate': 1.098e-05, - 'pair': 'ETH/BTC', - 'stake_amount': 0.001, - 'stop_loss_abs': 9.882e-06, - 'stop_loss_pct': -10.0, - 'stop_loss_ratio': -0.1, - 'stoploss_order_id': None, - 'stoploss_last_update': ANY, - 'stoploss_last_update_timestamp': ANY, - 'initial_stop_loss_abs': 9.882e-06, - 'initial_stop_loss_pct': -10.0, - 'initial_stop_loss_ratio': -0.1, - 'stoploss_current_dist': -1.1080000000000002e-06, - 'stoploss_current_dist_ratio': -0.10081893, - 'stoploss_current_dist_pct': -10.08, - 'stoploss_entry_dist': -0.00010475, - 'stoploss_entry_dist_ratio': -0.10448878, - 'trade_id': 1, - 'close_rate_requested': None, - 'current_rate': 1.099e-05, - 'fee_close': 0.0025, - 'fee_close_cost': None, - 'fee_close_currency': None, - 'fee_open': 0.0025, - 'fee_open_cost': None, - 'fee_open_currency': None, - 'open_date': ANY, - 'is_open': True, - 'max_rate': 1.099e-05, - 'min_rate': 1.098e-05, - 'open_order_id': None, - 'open_rate_requested': 1.098e-05, - 'open_trade_value': 0.0010025, - 'sell_reason': None, - 'sell_order_status': None, - 'strategy': 'DefaultStrategy', - 'timeframe': 5, - 'exchange': 'bittrex', - }] + assert len(rc.json()) == 1 + assert rc.json() == [{ + 'amount': 91.07468123, + 'amount_requested': 91.07468123, + 'base_currency': 'BTC', + 'close_date': None, + 'close_date_hum': None, + 'close_timestamp': None, + 'close_profit': None, + 'close_profit_pct': None, + 'close_profit_abs': None, + 'close_rate': None, + 'current_profit': -0.00408133, + 'current_profit_pct': -0.41, + 'current_profit_abs': -4.09e-06, + 'profit_ratio': -0.00408133, + 'profit_pct': -0.41, + 'profit_abs': -4.09e-06, + 'current_rate': 1.099e-05, + 'open_date': ANY, + 'open_date_hum': 'just now', + 'open_timestamp': ANY, + 'open_order': None, + 'open_rate': 1.098e-05, + 'pair': 'ETH/BTC', + 'stake_amount': 0.001, + 'stop_loss_abs': 9.882e-06, + 'stop_loss_pct': -10.0, + 'stop_loss_ratio': -0.1, + 'stoploss_order_id': None, + 'stoploss_last_update': ANY, + 'stoploss_last_update_timestamp': ANY, + 'initial_stop_loss_abs': 9.882e-06, + 'initial_stop_loss_pct': -10.0, + 'initial_stop_loss_ratio': -0.1, + 'stoploss_current_dist': -1.1080000000000002e-06, + 'stoploss_current_dist_ratio': -0.10081893, + 'stoploss_current_dist_pct': -10.08, + 'stoploss_entry_dist': -0.00010475, + 'stoploss_entry_dist_ratio': -0.10448878, + 'trade_id': 1, + 'close_rate_requested': None, + 'current_rate': 1.099e-05, + 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, + 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, + 'open_date': ANY, + 'is_open': True, + 'max_rate': 1.099e-05, + 'min_rate': 1.098e-05, + 'open_order_id': None, + 'open_rate_requested': 1.098e-05, + 'open_trade_value': 0.0010025, + 'sell_reason': None, + 'sell_order_status': None, + 'strategy': 'DefaultStrategy', + 'timeframe': 5, + 'exchange': 'bittrex', + }] def test_api_version(botclient): @@ -733,33 +736,33 @@ def test_api_blacklist(botclient, mocker): rc = client_get(client, f"{BASE_URI}/blacklist") assert_response(rc) # DOGE and HOT are not in the markets mock! - assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC"], - "blacklist_expanded": [], - "length": 2, - "method": ["StaticPairList"], - "errors": {}, - } + assert rc.json() == {"blacklist": ["DOGE/BTC", "HOT/BTC"], + "blacklist_expanded": [], + "length": 2, + "method": ["StaticPairList"], + "errors": {}, + } # Add ETH/BTC to blacklist rc = client_post(client, f"{BASE_URI}/blacklist", data='{"blacklist": ["ETH/BTC"]}') assert_response(rc) - assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC"], - "blacklist_expanded": ["ETH/BTC"], - "length": 3, - "method": ["StaticPairList"], - "errors": {}, - } + assert rc.json() == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC"], + "blacklist_expanded": ["ETH/BTC"], + "length": 3, + "method": ["StaticPairList"], + "errors": {}, + } rc = client_post(client, f"{BASE_URI}/blacklist", data='{"blacklist": ["XRP/.*"]}') assert_response(rc) - assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"], - "blacklist_expanded": ["ETH/BTC", "XRP/BTC"], - "length": 4, - "method": ["StaticPairList"], - "errors": {}, - } + assert rc.json() == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"], + "blacklist_expanded": ["ETH/BTC", "XRP/BTC"], + "length": 4, + "method": ["StaticPairList"], + "errors": {}, + } def test_api_whitelist(botclient): @@ -767,9 +770,11 @@ def test_api_whitelist(botclient): rc = client_get(client, f"{BASE_URI}/whitelist") assert_response(rc) - assert rc.json == {"whitelist": ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'], - "length": 4, - "method": ["StaticPairList"]} + assert rc.json() == { + "whitelist": ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'], + "length": 4, + "method": ["StaticPairList"] + } def test_api_forcebuy(botclient, mocker, fee): @@ -778,7 +783,7 @@ def test_api_forcebuy(botclient, mocker, fee): rc = client_post(client, f"{BASE_URI}/forcebuy", data='{"pair": "ETH/BTC"}') assert_response(rc, 502) - assert rc.json == {"error": "Error querying _forcebuy: Forcebuy not enabled."} + assert rc.json() == {"error": "Error querying /api/v1/forcebuy: Forcebuy not enabled."} # enable forcebuy ftbot.config['forcebuy_enable'] = True @@ -788,9 +793,9 @@ def test_api_forcebuy(botclient, mocker, fee): rc = client_post(client, f"{BASE_URI}/forcebuy", data='{"pair": "ETH/BTC"}') assert_response(rc) - assert rc.json == {"status": "Error buying pair ETH/BTC."} + assert rc.json() == {"status": "Error buying pair ETH/BTC."} - # Test creating trae + # Test creating trade fbuy_mock = MagicMock(return_value=Trade( pair='ETH/ETH', amount=1, @@ -810,53 +815,54 @@ def test_api_forcebuy(botclient, mocker, fee): rc = client_post(client, f"{BASE_URI}/forcebuy", data='{"pair": "ETH/BTC"}') assert_response(rc) - assert rc.json == {'amount': 1, - 'amount_requested': 1, - 'trade_id': None, - 'close_date': None, - 'close_date_hum': None, - 'close_timestamp': None, - 'close_rate': 0.265441, - 'open_date': ANY, - 'open_date_hum': 'just now', - 'open_timestamp': ANY, - 'open_rate': 0.245441, - 'pair': 'ETH/ETH', - 'stake_amount': 1, - 'stop_loss_abs': None, - 'stop_loss_pct': None, - 'stop_loss_ratio': None, - 'stoploss_order_id': None, - 'stoploss_last_update': None, - 'stoploss_last_update_timestamp': None, - 'initial_stop_loss_abs': None, - 'initial_stop_loss_pct': None, - 'initial_stop_loss_ratio': None, - 'close_profit': None, - 'close_profit_pct': None, - 'close_profit_abs': None, - 'close_rate_requested': None, - 'profit_ratio': None, - 'profit_pct': None, - 'profit_abs': None, - 'fee_close': 0.0025, - 'fee_close_cost': None, - 'fee_close_currency': None, - 'fee_open': 0.0025, - 'fee_open_cost': None, - 'fee_open_currency': None, - 'is_open': False, - 'max_rate': None, - 'min_rate': None, - 'open_order_id': '123456', - 'open_rate_requested': None, - 'open_trade_value': 0.24605460, - 'sell_reason': None, - 'sell_order_status': None, - 'strategy': None, - 'timeframe': None, - 'exchange': 'bittrex', - } + assert rc.json() == { + 'amount': 1, + 'amount_requested': 1, + 'trade_id': None, + 'close_date': None, + 'close_date_hum': None, + 'close_timestamp': None, + 'close_rate': 0.265441, + 'open_date': ANY, + 'open_date_hum': 'just now', + 'open_timestamp': ANY, + 'open_rate': 0.245441, + 'pair': 'ETH/ETH', + 'stake_amount': 1, + 'stop_loss_abs': None, + 'stop_loss_pct': None, + 'stop_loss_ratio': None, + 'stoploss_order_id': None, + 'stoploss_last_update': None, + 'stoploss_last_update_timestamp': None, + 'initial_stop_loss_abs': None, + 'initial_stop_loss_pct': None, + 'initial_stop_loss_ratio': None, + 'close_profit': None, + 'close_profit_pct': None, + 'close_profit_abs': None, + 'close_rate_requested': None, + 'profit_ratio': None, + 'profit_pct': None, + 'profit_abs': None, + 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, + 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, + 'is_open': False, + 'max_rate': None, + 'min_rate': None, + 'open_order_id': '123456', + 'open_rate_requested': None, + 'open_trade_value': 0.24605460, + 'sell_reason': None, + 'sell_order_status': None, + 'strategy': None, + 'timeframe': None, + 'exchange': 'bittrex', + } def test_api_forcesell(botclient, mocker, ticker, fee, markets): @@ -873,14 +879,14 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets): rc = client_post(client, f"{BASE_URI}/forcesell", data='{"tradeid": "1"}') assert_response(rc, 502) - assert rc.json == {"error": "Error querying _forcesell: invalid argument"} + assert rc.json() == {"error": "Error querying /api/v1/forcesell: invalid argument"} ftbot.enter_positions() rc = client_post(client, f"{BASE_URI}/forcesell", data='{"tradeid": "1"}') assert_response(rc) - assert rc.json == {'result': 'Created sell order for trade 1.'} + assert rc.json() == {'result': 'Created sell order for trade 1.'} def test_api_pair_candles(botclient, ohlcv_history): From e23898d17bffd9d3c1d3a59e8a0ab059d4c41c85 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Dec 2020 17:48:19 +0100 Subject: [PATCH 051/183] Improve some tests --- freqtrade/rpc/api_server2/api_models.py | 1 - freqtrade/rpc/api_server2/api_v1.py | 6 ++++-- tests/rpc/test_rpc_apiserver.py | 25 +++++++++++-------------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_models.py b/freqtrade/rpc/api_server2/api_models.py index a30973845..bcb3c280e 100644 --- a/freqtrade/rpc/api_server2/api_models.py +++ b/freqtrade/rpc/api_server2/api_models.py @@ -156,7 +156,6 @@ class WhitelistResponse(BaseModel): method: List[str] - class DeleteTrade(BaseModel): cancel_order_count: int result: str diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server2/api_v1.py index 00964f162..8cfc85a55 100644 --- a/freqtrade/rpc/api_server2/api_v1.py +++ b/freqtrade/rpc/api_server2/api_v1.py @@ -19,17 +19,19 @@ router = APIRouter() @router_public.get('/ping', response_model=Ping) def ping(): - """simple ping version""" + """simple ping""" return {"status": "pong"} @router.get('/version', response_model=Version, tags=['info']) def version(): + """ Bot Version info""" return {"version": __version__} @router.get('/balance', response_model=Balances, tags=['info']) def balance(rpc: RPC = Depends(get_rpc), config=Depends(get_config)): + """Account Balances""" return rpc._rpc_balance(config['stake_currency'], config.get('fiat_display_currency', ''),) @@ -71,7 +73,7 @@ def status(rpc: RPC = Depends(get_rpc)): # TODO: Missing response model -@router.get('/trades', tags=['info']) +@router.get('/trades', tags=['info', 'trading']) def trades(limit: Optional[int] = 0, rpc: RPC = Depends(get_rpc)): return rpc._rpc_trade_history(limit) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 49ca4aa9c..6ff1b3e95 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -80,10 +80,7 @@ def test_api_not_found(botclient): rc = client_post(client, f"{BASE_URI}/invalid_url") assert_response(rc, 404) - assert rc.json() == {"status": "error", - "reason": f"There's no API call for http://localhost{BASE_URI}/invalid_url.", - "code": 404 - } + assert rc.json() == {"detail": "Not Found"} def test_api_unauthorized(botclient): @@ -137,13 +134,12 @@ def test_api_token_refresh(botclient): rc = client_post(client, f"{BASE_URI}/token/login") assert_response(rc) rc = client.post(f"{BASE_URI}/token/refresh", - content_type="application/json", data=None, - headers={'Authorization': f'Bearer {rc.json["refresh_token"]}', + headers={'Authorization': f'Bearer {rc.json()["refresh_token"]}', 'Origin': 'http://example.com'}) assert_response(rc) - assert 'access_token' in rc.json - assert 'refresh_token' not in rc.json + assert 'access_token' in rc.json() + assert 'refresh_token' not in rc.json() def test_api_stop_workflow(botclient): @@ -151,24 +147,24 @@ def test_api_stop_workflow(botclient): assert ftbot.state == State.RUNNING rc = client_post(client, f"{BASE_URI}/stop") assert_response(rc) - assert rc.json == {'status': 'stopping trader ...'} + assert rc.json() == {'status': 'stopping trader ...'} assert ftbot.state == State.STOPPED # Stop bot again rc = client_post(client, f"{BASE_URI}/stop") assert_response(rc) - assert rc.json == {'status': 'already stopped'} + assert rc.json() == {'status': 'already stopped'} # Start bot rc = client_post(client, f"{BASE_URI}/start") assert_response(rc) - assert rc.json == {'status': 'starting trader ...'} + assert rc.json() == {'status': 'starting trader ...'} assert ftbot.state == State.RUNNING # Call start again rc = client_post(client, f"{BASE_URI}/start") assert_response(rc) - assert rc.json == {'status': 'already running'} + assert rc.json() == {'status': 'already running'} def test_api__init__(default_conf, mocker): @@ -182,7 +178,7 @@ def test_api__init__(default_conf, mocker): "password": "testPass", }}) mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) - mocker.patch('freqtrade.rpc.api_server.ApiServer.run', MagicMock()) + mocker.patch('freqtrade.rpc.api_server2.webserver.ApiServer.start_api', MagicMock()) apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) assert apiserver._config == default_conf @@ -255,7 +251,7 @@ def test_api_cleanup(default_conf, mocker, caplog): mocker.patch('freqtrade.rpc.api_server.make_server', MagicMock()) apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) - apiserver.run() + apiserver.run_api() stop_mock = MagicMock() stop_mock.shutdown = MagicMock() apiserver.srv = stop_mock @@ -655,6 +651,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets): trades = Trade.get_open_trades() trades[0].open_order_id = None ftbot.exit_positions(trades) + Trade.session.flush() rc = client_get(client, f"{BASE_URI}/status") assert_response(rc) From 9350f505bcee3bb5855d4d7da67c9f210fee3528 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 26 Dec 2020 20:05:27 +0100 Subject: [PATCH 052/183] Implement missing methods --- freqtrade/rpc/api_server2/api_models.py | 22 ++++- freqtrade/rpc/api_server2/api_v1.py | 88 ++++++++++++++++++- freqtrade/rpc/api_server2/webserver.py | 3 +- tests/rpc/test_rpc_apiserver.py | 110 ++++++++++++------------ 4 files changed, 162 insertions(+), 61 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_models.py b/freqtrade/rpc/api_server2/api_models.py index bcb3c280e..2e96dca40 100644 --- a/freqtrade/rpc/api_server2/api_models.py +++ b/freqtrade/rpc/api_server2/api_models.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union from datetime import date from pydantic import BaseModel @@ -161,3 +161,23 @@ class DeleteTrade(BaseModel): result: str result_msg: str trade_id: int + + +class PlotConfig(BaseModel): + main_plot: Optional[Dict[str, Any]] + subplots: Optional[Dict[str, Any]] + + +class StrategyListResponse(BaseModel): + strategies: List[str] + + +class StrategyResponse(BaseModel): + strategy: str + code: str + + +class AvailablePairs(BaseModel): + length: int + pairs: List[str] + pair_interval: List[List[str]] diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server2/api_v1.py index 8cfc85a55..8389b7336 100644 --- a/freqtrade/rpc/api_server2/api_v1.py +++ b/freqtrade/rpc/api_server2/api_v1.py @@ -1,13 +1,21 @@ -from typing import List, Optional +from copy import deepcopy +from freqtrade.constants import USERPATH_STRATEGIES +from typing import Dict, List, Optional, Union +from pathlib import Path from fastapi import APIRouter, Depends +from fastapi.exceptions import HTTPException from freqtrade import __version__ +from freqtrade.data.history import get_datahandler +from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC from freqtrade.rpc.rpc import RPCException -from .api_models import (Balances, BlacklistPayload, BlacklistResponse, Count, Daily, DeleteTrade, ForceBuyPayload, ForceSellPayload, Locks, Logs, PerformanceEntry, Ping, Profit, ResultMsg, Stats, - StatusMsg, Version, WhitelistResponse) +from .api_models import (AvailablePairs, Balances, BlacklistPayload, BlacklistResponse, Count, + Daily, DeleteTrade, ForceBuyPayload, ForceSellPayload, Locks, Logs, + PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, Stats, StatusMsg, StrategyListResponse, StrategyResponse, Version, + WhitelistResponse) from .deps import get_config, get_rpc @@ -154,3 +162,77 @@ def stop_buy(rpc: RPC = Depends(get_rpc)): @router.post('/reload_config', response_model=StatusMsg, tags=['botcontrol']) def reload_config(rpc: RPC = Depends(get_rpc)): return rpc._rpc_reload_config() + + +# TODO: Missing response model +@router.get('/pair_candles', tags=['candle data']) +def pair_candles(pair: str, timeframe: str, limit: Optional[int], rpc=Depends(get_rpc)): + return rpc._rpc_analysed_dataframe(pair, timeframe, limit) + + +# TODO: Missing response model +@router.get('/pair_history', tags=['candle data']) +def pair_history(pair: str, timeframe: str, timerange: str, strategy: str, + config=Depends(get_config)): + config = deepcopy(config) + config.update({ + 'strategy': strategy, + }) + return RPC._rpc_analysed_history_full(config, pair, timeframe, timerange) + + +@router.get('/plot_config', response_model=Union[Dict, PlotConfig], tags=['candle data']) +def plot_config(rpc=Depends(get_rpc)): + return rpc._rpc_plot_config() + + +@router.get('/strategies', response_model=StrategyListResponse, tags=['strategy']) +def list_strategies(config=Depends(get_config)): + directory = Path(config.get( + 'strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES)) + from freqtrade.resolvers.strategy_resolver import StrategyResolver + strategies = StrategyResolver.search_all_objects(directory, False) + strategies = sorted(strategies, key=lambda x: x['name']) + + return {'strategies': [x['name'] for x in strategies]} + + +@router.get('/strategy/{strategy}', response_model=StrategyResponse, tags=['strategy']) +def get_strategy(strategy: str, rpc: RPC = Depends(get_rpc), config=Depends(get_config)): + + config = deepcopy(config) + from freqtrade.resolvers.strategy_resolver import StrategyResolver + try: + strategy_obj = StrategyResolver._load_strategy(strategy, config, + extra_dir=config.get('strategy_path')) + except OperationalException: + raise HTTPException(status_code=404, detail='Strategy not found') + + return { + 'strategy': strategy_obj.get_strategy_name(), + 'code': strategy_obj.__source__, + } + + +@router.get('/available_pairs', response_model=AvailablePairs, tags=['candle data']) +def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Optional[str] = None, + config=Depends(get_config)): + + dh = get_datahandler(config['datadir'], config.get('dataformat_ohlcv', None)) + + pair_interval = dh.ohlcv_get_available_data(config['datadir']) + + if timeframe: + pair_interval = [pair for pair in pair_interval if pair[1] == timeframe] + if stake_currency: + pair_interval = [pair for pair in pair_interval if pair[0].endswith(stake_currency)] + pair_interval = sorted(pair_interval, key=lambda x: x[0]) + + pairs = list({x[0] for x in pair_interval}) + + result = { + 'length': len(pairs), + 'pairs': pairs, + 'pair_interval': pair_interval, + } + return result diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server2/webserver.py index 45e695151..f54845535 100644 --- a/freqtrade/rpc/api_server2/webserver.py +++ b/freqtrade/rpc/api_server2/webserver.py @@ -1,11 +1,10 @@ import logging from typing import Any, Dict, Optional -from starlette.responses import JSONResponse import uvicorn from fastapi import Depends, FastAPI -from fastapi.exceptions import HTTPException from fastapi.middleware.cors import CORSMiddleware +from starlette.responses import JSONResponse from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 6ff1b3e95..4ebe63f46 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -894,22 +894,22 @@ def test_api_pair_candles(botclient, ohlcv_history): # No pair rc = client_get(client, f"{BASE_URI}/pair_candles?limit={amount}&timeframe={timeframe}") - assert_response(rc, 400) + assert_response(rc, 422) # No timeframe rc = client_get(client, f"{BASE_URI}/pair_candles?pair=XRP%2FBTC") - assert_response(rc, 400) + assert_response(rc, 422) rc = client_get(client, f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}") assert_response(rc) - assert 'columns' in rc.json - assert 'data_start_ts' in rc.json - assert 'data_start' in rc.json - assert 'data_stop' in rc.json - assert 'data_stop_ts' in rc.json - assert len(rc.json['data']) == 0 + assert 'columns' in rc.json() + assert 'data_start_ts' in rc.json() + assert 'data_start' in rc.json() + assert 'data_stop' in rc.json() + assert 'data_stop_ts' in rc.json() + assert len(rc.json()['data']) == 0 ohlcv_history['sma'] = ohlcv_history['close'].rolling(2).mean() ohlcv_history['buy'] = 0 ohlcv_history.loc[1, 'buy'] = 1 @@ -920,28 +920,28 @@ def test_api_pair_candles(botclient, ohlcv_history): rc = client_get(client, f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}") assert_response(rc) - assert 'strategy' in rc.json - assert rc.json['strategy'] == 'DefaultStrategy' - assert 'columns' in rc.json - assert 'data_start_ts' in rc.json - assert 'data_start' in rc.json - assert 'data_stop' in rc.json - assert 'data_stop_ts' in rc.json - assert rc.json['data_start'] == '2017-11-26 08:50:00+00:00' - assert rc.json['data_start_ts'] == 1511686200000 - assert rc.json['data_stop'] == '2017-11-26 09:00:00+00:00' - assert rc.json['data_stop_ts'] == 1511686800000 - assert isinstance(rc.json['columns'], list) - assert rc.json['columns'] == ['date', 'open', 'high', + assert 'strategy' in rc.json() + assert rc.json()['strategy'] == 'DefaultStrategy' + assert 'columns' in rc.json() + assert 'data_start_ts' in rc.json() + assert 'data_start' in rc.json() + assert 'data_stop' in rc.json() + assert 'data_stop_ts' in rc.json() + assert rc.json()['data_start'] == '2017-11-26 08:50:00+00:00' + assert rc.json()['data_start_ts'] == 1511686200000 + assert rc.json()['data_stop'] == '2017-11-26 09:00:00+00:00' + assert rc.json()['data_stop_ts'] == 1511686800000 + assert isinstance(rc.json()['columns'], list) + assert rc.json()['columns'] == ['date', 'open', 'high', 'low', 'close', 'volume', 'sma', 'buy', 'sell', '__date_ts', '_buy_signal_open', '_sell_signal_open'] - assert 'pair' in rc.json - assert rc.json['pair'] == 'XRP/BTC' + assert 'pair' in rc.json() + assert rc.json()['pair'] == 'XRP/BTC' - assert 'data' in rc.json - assert len(rc.json['data']) == amount + assert 'data' in rc.json() + assert len(rc.json()['data']) == amount - assert (rc.json['data'] == + assert (rc.json()['data'] == [['2017-11-26 08:50:00', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, None, 0, 0, 1511686200000, None, None], ['2017-11-26 08:55:00', 8.88e-05, 8.942e-05, 8.88e-05, @@ -960,41 +960,41 @@ def test_api_pair_history(botclient, ohlcv_history): rc = client_get(client, f"{BASE_URI}/pair_history?timeframe={timeframe}" "&timerange=20180111-20180112&strategy=DefaultStrategy") - assert_response(rc, 400) + assert_response(rc, 422) # No Timeframe rc = client_get(client, f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC" "&timerange=20180111-20180112&strategy=DefaultStrategy") - assert_response(rc, 400) + assert_response(rc, 422) # No timerange rc = client_get(client, f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" "&strategy=DefaultStrategy") - assert_response(rc, 400) + assert_response(rc, 422) # No strategy rc = client_get(client, f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" "&timerange=20180111-20180112") - assert_response(rc, 400) + assert_response(rc, 422) # Working rc = client_get(client, f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" "&timerange=20180111-20180112&strategy=DefaultStrategy") assert_response(rc, 200) - assert rc.json['length'] == 289 - assert len(rc.json['data']) == rc.json['length'] - assert 'columns' in rc.json - assert 'data' in rc.json - assert rc.json['pair'] == 'UNITTEST/BTC' - assert rc.json['strategy'] == 'DefaultStrategy' - assert rc.json['data_start'] == '2018-01-11 00:00:00+00:00' - assert rc.json['data_start_ts'] == 1515628800000 - assert rc.json['data_stop'] == '2018-01-12 00:00:00+00:00' - assert rc.json['data_stop_ts'] == 1515715200000 + assert rc.json()['length'] == 289 + assert len(rc.json()['data']) == rc.json()['length'] + assert 'columns' in rc.json() + assert 'data' in rc.json() + assert rc.json()['pair'] == 'UNITTEST/BTC' + assert rc.json()['strategy'] == 'DefaultStrategy' + assert rc.json()['data_start'] == '2018-01-11 00:00:00+00:00' + assert rc.json()['data_start_ts'] == 1515628800000 + assert rc.json()['data_stop'] == '2018-01-12 00:00:00+00:00' + assert rc.json()['data_stop_ts'] == 1515715200000 def test_api_plot_config(botclient): @@ -1002,14 +1002,14 @@ def test_api_plot_config(botclient): rc = client_get(client, f"{BASE_URI}/plot_config") assert_response(rc) - assert rc.json == {} + assert rc.json() == {} ftbot.strategy.plot_config = {'main_plot': {'sma': {}}, 'subplots': {'RSI': {'rsi': {'color': 'red'}}}} rc = client_get(client, f"{BASE_URI}/plot_config") assert_response(rc) - assert rc.json == ftbot.strategy.plot_config - assert isinstance(rc.json['main_plot'], dict) + assert rc.json() == ftbot.strategy.plot_config + assert isinstance(rc.json()['main_plot'], dict) def test_api_strategies(botclient): @@ -1018,7 +1018,7 @@ def test_api_strategies(botclient): rc = client_get(client, f"{BASE_URI}/strategies") assert_response(rc) - assert rc.json == {'strategies': ['DefaultStrategy', 'TestStrategyLegacy']} + assert rc.json() == {'strategies': ['DefaultStrategy', 'TestStrategyLegacy']} def test_api_strategy(botclient): @@ -1027,10 +1027,10 @@ def test_api_strategy(botclient): rc = client_get(client, f"{BASE_URI}/strategy/DefaultStrategy") assert_response(rc) - assert rc.json['strategy'] == 'DefaultStrategy' + assert rc.json()['strategy'] == 'DefaultStrategy' data = (Path(__file__).parents[1] / "strategy/strats/default_strategy.py").read_text() - assert rc.json['code'] == data + assert rc.json()['code'] == data rc = client_get(client, f"{BASE_URI}/strategy/NoStrat") assert_response(rc, 404) @@ -1042,21 +1042,21 @@ def test_list_available_pairs(botclient): rc = client_get(client, f"{BASE_URI}/available_pairs") assert_response(rc) - assert rc.json['length'] == 12 - assert isinstance(rc.json['pairs'], list) + assert rc.json()['length'] == 12 + assert isinstance(rc.json()['pairs'], list) rc = client_get(client, f"{BASE_URI}/available_pairs?timeframe=5m") assert_response(rc) - assert rc.json['length'] == 12 + assert rc.json()['length'] == 12 rc = client_get(client, f"{BASE_URI}/available_pairs?stake_currency=ETH") assert_response(rc) - assert rc.json['length'] == 1 - assert rc.json['pairs'] == ['XRP/ETH'] - assert len(rc.json['pair_interval']) == 2 + assert rc.json()['length'] == 1 + assert rc.json()['pairs'] == ['XRP/ETH'] + assert len(rc.json()['pair_interval']) == 2 rc = client_get(client, f"{BASE_URI}/available_pairs?stake_currency=ETH&timeframe=5m") assert_response(rc) - assert rc.json['length'] == 1 - assert rc.json['pairs'] == ['XRP/ETH'] - assert len(rc.json['pair_interval']) == 1 + assert rc.json()['length'] == 1 + assert rc.json()['pairs'] == ['XRP/ETH'] + assert len(rc.json()['pair_interval']) == 1 From 9f873305ebe6a2a5d0e4382c1277b1d9a0ebabc8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Dec 2020 09:02:35 +0100 Subject: [PATCH 053/183] Improve response models --- freqtrade/rpc/api_server2/api_auth.py | 3 +-- freqtrade/rpc/api_server2/api_models.py | 28 ++++++++++++++++++++++++- freqtrade/rpc/api_server2/api_v1.py | 13 ++++++------ tests/rpc/test_rpc_apiserver.py | 4 ++-- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_auth.py b/freqtrade/rpc/api_server2/api_auth.py index cb19d7637..599f6b53c 100644 --- a/freqtrade/rpc/api_server2/api_auth.py +++ b/freqtrade/rpc/api_server2/api_auth.py @@ -2,10 +2,9 @@ import secrets from datetime import datetime, timedelta import jwt -from fastapi import APIRouter, Depends, HTTPException, Request, status +from fastapi import APIRouter, Depends, HTTPException, status from fastapi.security import OAuth2PasswordBearer from fastapi.security.http import HTTPBasic, HTTPBasicCredentials -from pydantic import BaseModel from freqtrade.rpc.api_server2.api_models import AccessAndRefreshToken, AccessToken diff --git a/freqtrade/rpc/api_server2/api_models.py b/freqtrade/rpc/api_server2/api_models.py index 2e96dca40..c9e4ee5cc 100644 --- a/freqtrade/rpc/api_server2/api_models.py +++ b/freqtrade/rpc/api_server2/api_models.py @@ -1,7 +1,10 @@ +from datetime import date, datetime from typing import Any, Dict, List, Optional, Union -from datetime import date + from pydantic import BaseModel +from freqtrade.constants import DATETIME_PRINT_FORMAT + class Ping(BaseModel): status: str @@ -181,3 +184,26 @@ class AvailablePairs(BaseModel): length: int pairs: List[str] pair_interval: List[List[str]] + + +class PairHistory(BaseModel): + strategy: str + pair: str + timeframe: str + timeframe_ms: int + columns: List[str] + data: List[Any] + length: int + buy_signals: int + sell_signals: int + last_analyzed: datetime + last_analyzed_ts: int + data_start_ts: int + data_start: str + data_stop: str + data_stop_ts: int + + class Config: + json_encoders = { + datetime: lambda v: v.strftime(DATETIME_PRINT_FORMAT), + } diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server2/api_v1.py index 8389b7336..21c525850 100644 --- a/freqtrade/rpc/api_server2/api_v1.py +++ b/freqtrade/rpc/api_server2/api_v1.py @@ -1,12 +1,12 @@ from copy import deepcopy -from freqtrade.constants import USERPATH_STRATEGIES -from typing import Dict, List, Optional, Union from pathlib import Path +from typing import Dict, List, Optional, Union from fastapi import APIRouter, Depends from fastapi.exceptions import HTTPException from freqtrade import __version__ +from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.data.history import get_datahandler from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC @@ -14,7 +14,8 @@ from freqtrade.rpc.rpc import RPCException from .api_models import (AvailablePairs, Balances, BlacklistPayload, BlacklistResponse, Count, Daily, DeleteTrade, ForceBuyPayload, ForceSellPayload, Locks, Logs, - PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, Stats, StatusMsg, StrategyListResponse, StrategyResponse, Version, + PairHistory, PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, Stats, + StatusMsg, StrategyListResponse, StrategyResponse, Version, WhitelistResponse) from .deps import get_config, get_rpc @@ -164,14 +165,12 @@ def reload_config(rpc: RPC = Depends(get_rpc)): return rpc._rpc_reload_config() -# TODO: Missing response model -@router.get('/pair_candles', tags=['candle data']) +@router.get('/pair_candles', response_model=PairHistory, tags=['candle data']) def pair_candles(pair: str, timeframe: str, limit: Optional[int], rpc=Depends(get_rpc)): return rpc._rpc_analysed_dataframe(pair, timeframe, limit) -# TODO: Missing response model -@router.get('/pair_history', tags=['candle data']) +@router.get('/pair_history', response_model=PairHistory, tags=['candle data']) def pair_history(pair: str, timeframe: str, timerange: str, strategy: str, config=Depends(get_config)): config = deepcopy(config) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 4ebe63f46..7d1100fb8 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -933,8 +933,8 @@ def test_api_pair_candles(botclient, ohlcv_history): assert rc.json()['data_stop_ts'] == 1511686800000 assert isinstance(rc.json()['columns'], list) assert rc.json()['columns'] == ['date', 'open', 'high', - 'low', 'close', 'volume', 'sma', 'buy', 'sell', - '__date_ts', '_buy_signal_open', '_sell_signal_open'] + 'low', 'close', 'volume', 'sma', 'buy', 'sell', + '__date_ts', '_buy_signal_open', '_sell_signal_open'] assert 'pair' in rc.json() assert rc.json()['pair'] == 'XRP/BTC' From 54a50b1fb4b3dd0fc405271db6c620a7bde412d4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Dec 2020 10:56:19 +0100 Subject: [PATCH 054/183] Fix some tests --- freqtrade/rpc/api_server2/uvicorn_threaded.py | 3 ++ freqtrade/rpc/api_server2/webserver.py | 33 ++++++++++---- tests/rpc/test_rpc_apiserver.py | 44 +++++++++---------- tests/rpc/test_rpc_manager.py | 4 +- 4 files changed, 51 insertions(+), 33 deletions(-) diff --git a/freqtrade/rpc/api_server2/uvicorn_threaded.py b/freqtrade/rpc/api_server2/uvicorn_threaded.py index 7c8804fd3..ce7089bed 100644 --- a/freqtrade/rpc/api_server2/uvicorn_threaded.py +++ b/freqtrade/rpc/api_server2/uvicorn_threaded.py @@ -10,6 +10,9 @@ class UvicornServer(uvicorn.Server): Multithreaded server - as found in https://github.com/encode/uvicorn/issues/742 """ def install_signal_handlers(self): + """ + In the parent implementation, this starts the thread, therefore we must patch it away here. + """ pass @contextlib.contextmanager diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server2/webserver.py index f54845535..b3e6eb0dc 100644 --- a/freqtrade/rpc/api_server2/webserver.py +++ b/freqtrade/rpc/api_server2/webserver.py @@ -1,4 +1,5 @@ import logging +from ipaddress import IPv4Address from typing import Any, Dict, Optional import uvicorn @@ -16,7 +17,7 @@ logger = logging.getLogger(__name__) class ApiServer(RPCHandler): - _rpc: Optional[RPC] = None + _rpc: RPC = None _config: Dict[str, Any] = {} def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: @@ -34,6 +35,7 @@ class ApiServer(RPCHandler): def cleanup(self) -> None: """ Cleanup pending module resources """ if self._server: + logger.info("Stopping API Server") self._server.cleanup() def send_msg(self, msg: Dict[str, str]) -> None: @@ -71,11 +73,26 @@ class ApiServer(RPCHandler): """ Start API ... should be run in thread. """ - uvconfig = uvicorn.Config(self.app, - port=self._config['api_server'].get('listen_port', 8080), - host=self._config['api_server'].get( - 'listen_ip_address', '127.0.0.1'), - access_log=True) - self._server = UvicornServer(uvconfig) + rest_ip = self._config['api_server']['listen_ip_address'] + rest_port = self._config['api_server']['listen_port'] - self._server.run_in_thread() + logger.info(f'Starting HTTP Server at {rest_ip}:{rest_port}') + if not IPv4Address(rest_ip).is_loopback: + logger.warning("SECURITY WARNING - Local Rest Server listening to external connections") + logger.warning("SECURITY WARNING - This is insecure please set to your loopback," + "e.g 127.0.0.1 in config.json") + + if not self._config['api_server'].get('password'): + logger.warning("SECURITY WARNING - No password for local REST Server defined. " + "Please make sure that this is intentional!") + + logger.info('Starting Local Rest Server.') + uvconfig = uvicorn.Config(self.app, + port=rest_port, + host=rest_ip, + access_log=True) + try: + self._server = UvicornServer(uvconfig) + self._server.run_in_thread() + except Exception: + logger.exception("Api server failed to start.") diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 7d1100fb8..8017293b4 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -5,17 +5,18 @@ Unit test file for rpc/api_server.py from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import ANY, MagicMock, PropertyMock +from fastapi.applications import FastAPI import pytest from fastapi.testclient import TestClient -from flask import Flask +from fastapi import FastAPI from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ from freqtrade.loggers import setup_logging, setup_logging_pre from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC -from freqtrade.rpc.api_server import BASE_URI # , ApiServer +from freqtrade.rpc.api_server111 import BASE_URI # , ApiServer from freqtrade.rpc.api_server2 import ApiServer from freqtrade.state import RunMode, State from tests.conftest import create_mock_trades, get_patched_freqtradebot, log_has, patch_get_signal @@ -191,20 +192,19 @@ def test_api_run(default_conf, mocker, caplog): "password": "testPass", }}) mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) - mocker.patch('freqtrade.rpc.api_server.threading.Thread', MagicMock()) server_mock = MagicMock() - mocker.patch('freqtrade.rpc.api_server.make_server', server_mock) + mocker.patch('freqtrade.rpc.api_server2.webserver.UvicornServer', server_mock) apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) - assert apiserver._config == default_conf - apiserver.run() assert server_mock.call_count == 1 - assert server_mock.call_args_list[0][0][0] == "127.0.0.1" - assert server_mock.call_args_list[0][0][1] == 8080 - assert isinstance(server_mock.call_args_list[0][0][2], Flask) - assert hasattr(apiserver, "srv") + assert apiserver._config == default_conf + apiserver.start_api() + assert server_mock.call_count == 2 + assert server_mock.call_args_list[0][0][0].host == "127.0.0.1" + assert server_mock.call_args_list[0][0][0].port == 8080 + assert isinstance(server_mock.call_args_list[0][0][0].app, FastAPI) assert log_has("Starting HTTP Server at 127.0.0.1:8080", caplog) assert log_has("Starting Local Rest Server.", caplog) @@ -217,12 +217,12 @@ def test_api_run(default_conf, mocker, caplog): "listen_port": 8089, "password": "", }}) - apiserver.run() + apiserver.start_api() assert server_mock.call_count == 1 - assert server_mock.call_args_list[0][0][0] == "0.0.0.0" - assert server_mock.call_args_list[0][0][1] == 8089 - assert isinstance(server_mock.call_args_list[0][0][2], Flask) + assert server_mock.call_args_list[0][0][0].host == "0.0.0.0" + assert server_mock.call_args_list[0][0][0].port == 8089 + assert isinstance(server_mock.call_args_list[0][0][0].app, FastAPI) assert log_has("Starting HTTP Server at 0.0.0.0:8089", caplog) assert log_has("Starting Local Rest Server.", caplog) assert log_has("SECURITY WARNING - Local Rest Server listening to external connections", @@ -234,8 +234,8 @@ def test_api_run(default_conf, mocker, caplog): # Test crashing flask caplog.clear() - mocker.patch('freqtrade.rpc.api_server.make_server', MagicMock(side_effect=Exception)) - apiserver.run() + mocker.patch('freqtrade.rpc.api_server2.webserver.UvicornServer', MagicMock(side_effect=Exception)) + apiserver.start_api() assert log_has("Api server failed to start.", caplog) @@ -247,17 +247,15 @@ def test_api_cleanup(default_conf, mocker, caplog): "password": "testPass", }}) mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) - mocker.patch('freqtrade.rpc.api_server.threading.Thread', MagicMock()) - mocker.patch('freqtrade.rpc.api_server.make_server', MagicMock()) + + server_mock = MagicMock() + server_mock.cleanup = MagicMock() + mocker.patch('freqtrade.rpc.api_server2.webserver.UvicornServer', server_mock) apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) - apiserver.run_api() - stop_mock = MagicMock() - stop_mock.shutdown = MagicMock() - apiserver.srv = stop_mock apiserver.cleanup() - assert stop_mock.shutdown.call_count == 1 + assert apiserver._server.cleanup.call_count == 1 assert log_has("Stopping API Server", caplog) diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index 06706120f..e63d629b8 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -160,7 +160,7 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: def test_init_apiserver_disabled(mocker, default_conf, caplog) -> None: caplog.set_level(logging.DEBUG) run_mock = MagicMock() - mocker.patch('freqtrade.rpc.api_server.ApiServer.run', run_mock) + mocker.patch('freqtrade.rpc.api_server2.ApiServer.start_api', run_mock) default_conf['telegram']['enabled'] = False rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) @@ -172,7 +172,7 @@ def test_init_apiserver_disabled(mocker, default_conf, caplog) -> None: def test_init_apiserver_enabled(mocker, default_conf, caplog) -> None: caplog.set_level(logging.DEBUG) run_mock = MagicMock() - mocker.patch('freqtrade.rpc.api_server.ApiServer.run', run_mock) + mocker.patch('freqtrade.rpc.api_server2.ApiServer.start_api', run_mock) default_conf["telegram"]["enabled"] = False default_conf["api_server"] = {"enabled": True, From 776ce57f55f912f1c7c4914982b2125b55a80327 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Dec 2020 10:59:17 +0100 Subject: [PATCH 055/183] Remove api_server --- freqtrade/rpc/api_server.py | 665 ------------------------- freqtrade/rpc/api_server2/webserver.py | 2 +- tests/rpc/test_rpc_apiserver.py | 5 +- 3 files changed, 3 insertions(+), 669 deletions(-) delete mode 100644 freqtrade/rpc/api_server.py diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py deleted file mode 100644 index b489586c8..000000000 --- a/freqtrade/rpc/api_server.py +++ /dev/null @@ -1,665 +0,0 @@ -import logging -import threading -from copy import deepcopy -from datetime import date, datetime -from ipaddress import IPv4Address -from pathlib import Path -from typing import Any, Callable, Dict - -from arrow import Arrow -from flask import Flask, jsonify, request -from flask.json import JSONEncoder -from flask_cors import CORS -from flask_jwt_extended import (JWTManager, create_access_token, create_refresh_token, - get_jwt_identity, jwt_refresh_token_required, - verify_jwt_in_request_optional) -from werkzeug.security import safe_str_cmp -from werkzeug.serving import make_server - -from freqtrade.__init__ import __version__ -from freqtrade.constants import DATETIME_PRINT_FORMAT, USERPATH_STRATEGIES -from freqtrade.exceptions import OperationalException -from freqtrade.persistence import Trade -from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler - - -logger = logging.getLogger(__name__) - -BASE_URI = "/api/v1" - - -class FTJSONEncoder(JSONEncoder): - def default(self, obj): - try: - if isinstance(obj, Arrow): - return obj.for_json() - elif isinstance(obj, datetime): - return obj.strftime(DATETIME_PRINT_FORMAT) - elif isinstance(obj, date): - return obj.strftime("%Y-%m-%d") - iterable = iter(obj) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, obj) - - -# Type should really be Callable[[ApiServer, Any], Any], but that will create a circular dependency -def require_login(func: Callable[[Any, Any], Any]): - - def func_wrapper(obj, *args, **kwargs): - verify_jwt_in_request_optional() - auth = request.authorization - if get_jwt_identity() or auth and obj.check_auth(auth.username, auth.password): - return func(obj, *args, **kwargs) - else: - return jsonify({"error": "Unauthorized"}), 401 - - return func_wrapper - - -# Type should really be Callable[[ApiServer], Any], but that will create a circular dependency -def rpc_catch_errors(func: Callable[..., Any]): - - def func_wrapper(obj, *args, **kwargs): - - try: - return func(obj, *args, **kwargs) - except RPCException as e: - logger.exception("API Error calling %s: %s", func.__name__, e) - return obj.rest_error(f"Error querying {func.__name__}: {e}") - - return func_wrapper - - -def shutdown_session(exception=None): - # Remove scoped session - Trade.session.remove() - - -class ApiServer(RPCHandler): - """ - This class runs api server and provides rpc.rpc functionality to it - - This class starts a non-blocking thread the api server runs within - """ - - def check_auth(self, username, password): - return (safe_str_cmp(username, self._config['api_server'].get('username')) and - safe_str_cmp(password, self._config['api_server'].get('password'))) - - def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: - """ - Init the api server, and init the super class RPCHandler - :param rpc: instance of RPC Helper class - :param config: Configuration object - :return: None - """ - super().__init__(rpc, config) - - self.app = Flask(__name__) - self._cors = CORS(self.app, - resources={r"/api/*": { - "supports_credentials": True, - "origins": self._config['api_server'].get('CORS_origins', [])}} - ) - - # Setup the Flask-JWT-Extended extension - self.app.config['JWT_SECRET_KEY'] = self._config['api_server'].get( - 'jwt_secret_key', 'super-secret') - - self.jwt = JWTManager(self.app) - self.app.json_encoder = FTJSONEncoder - - self.app.teardown_appcontext(shutdown_session) - - # Register application handling - self.register_rest_rpc_urls() - - thread = threading.Thread(target=self.run, daemon=True) - thread.start() - - def cleanup(self) -> None: - logger.info("Stopping API Server") - self.srv.shutdown() - - def run(self): - """ - Method that runs flask app in its own thread forever. - Section to handle configuration and running of the Rest server - also to check and warn if not bound to a loopback, warn on security risk. - """ - rest_ip = self._config['api_server']['listen_ip_address'] - rest_port = self._config['api_server']['listen_port'] - - logger.info(f'Starting HTTP Server at {rest_ip}:{rest_port}') - if not IPv4Address(rest_ip).is_loopback: - logger.warning("SECURITY WARNING - Local Rest Server listening to external connections") - logger.warning("SECURITY WARNING - This is insecure please set to your loopback," - "e.g 127.0.0.1 in config.json") - - if not self._config['api_server'].get('password'): - logger.warning("SECURITY WARNING - No password for local REST Server defined. " - "Please make sure that this is intentional!") - - # Run the Server - logger.info('Starting Local Rest Server.') - try: - self.srv = make_server(rest_ip, rest_port, self.app) - self.srv.serve_forever() - except Exception: - logger.exception("Api server failed to start.") - logger.info('Local Rest Server started.') - - def send_msg(self, msg: Dict[str, str]) -> None: - """ - We don't push to endpoints at the moment. - Take a look at webhooks for that functionality. - """ - pass - - def rest_error(self, error_msg, error_code=502): - return jsonify({"error": error_msg}), error_code - - def register_rest_rpc_urls(self): - """ - Registers flask app URLs that are calls to functionality in rpc.rpc. - - First two arguments passed are /URL and 'Label' - Label can be used as a shortcut when refactoring - :return: - """ - self.app.register_error_handler(404, self.page_not_found) - - # Actions to control the bot - self.app.add_url_rule(f'{BASE_URI}/token/login', 'login', - view_func=self._token_login, methods=['POST']) - self.app.add_url_rule(f'{BASE_URI}/token/refresh', 'token_refresh', - view_func=self._token_refresh, methods=['POST']) - self.app.add_url_rule(f'{BASE_URI}/start', 'start', - view_func=self._start, methods=['POST']) - self.app.add_url_rule(f'{BASE_URI}/stop', 'stop', view_func=self._stop, methods=['POST']) - self.app.add_url_rule(f'{BASE_URI}/stopbuy', 'stopbuy', - view_func=self._stopbuy, methods=['POST']) - self.app.add_url_rule(f'{BASE_URI}/reload_config', 'reload_config', - view_func=self._reload_config, methods=['POST']) - # Info commands - self.app.add_url_rule(f'{BASE_URI}/balance', 'balance', - view_func=self._balance, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/count', 'count', view_func=self._count, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/locks', 'locks', view_func=self._locks, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/daily', 'daily', view_func=self._daily, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/edge', 'edge', view_func=self._edge, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/logs', 'log', view_func=self._get_logs, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/profit', 'profit', - view_func=self._profit, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/stats', 'stats', - view_func=self._stats, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/performance', 'performance', - view_func=self._performance, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/status', 'status', - view_func=self._status, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/version', 'version', - view_func=self._version, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/show_config', 'show_config', - view_func=self._show_config, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/ping', 'ping', - view_func=self._ping, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/trades', 'trades', - view_func=self._trades, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/trades/', 'trades_delete', - view_func=self._trades_delete, methods=['DELETE']) - - self.app.add_url_rule(f'{BASE_URI}/pair_candles', 'pair_candles', - view_func=self._analysed_candles, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/pair_history', 'pair_history', - view_func=self._analysed_history, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/plot_config', 'plot_config', - view_func=self._plot_config, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/strategies', 'strategies', - view_func=self._list_strategies, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/strategy/', 'strategy', - view_func=self._get_strategy, methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/available_pairs', 'pairs', - view_func=self._list_available_pairs, methods=['GET']) - - # Combined actions and infos - self.app.add_url_rule(f'{BASE_URI}/blacklist', 'blacklist', view_func=self._blacklist, - methods=['GET', 'POST']) - self.app.add_url_rule(f'{BASE_URI}/whitelist', 'whitelist', view_func=self._whitelist, - methods=['GET']) - self.app.add_url_rule(f'{BASE_URI}/forcebuy', 'forcebuy', - view_func=self._forcebuy, methods=['POST']) - self.app.add_url_rule(f'{BASE_URI}/forcesell', 'forcesell', view_func=self._forcesell, - methods=['POST']) - - @require_login - def page_not_found(self, error): - """ - Return "404 not found", 404. - """ - return jsonify({ - 'status': 'error', - 'reason': f"There's no API call for {request.base_url}.", - 'code': 404 - }), 404 - - @require_login - @rpc_catch_errors - def _token_login(self): - """ - Handler for /token/login - Returns a JWT token - """ - auth = request.authorization - if auth and self.check_auth(auth.username, auth.password): - keystuff = {'u': auth.username} - ret = { - 'access_token': create_access_token(identity=keystuff), - 'refresh_token': create_refresh_token(identity=keystuff), - } - return jsonify(ret) - - return jsonify({"error": "Unauthorized"}), 401 - - @jwt_refresh_token_required - @rpc_catch_errors - def _token_refresh(self): - """ - Handler for /token/refresh - Returns a JWT token based on a JWT refresh token - """ - current_user = get_jwt_identity() - new_token = create_access_token(identity=current_user, fresh=False) - - ret = {'access_token': new_token} - return jsonify(ret) - - @require_login - @rpc_catch_errors - def _start(self): - """ - Handler for /start. - Starts TradeThread in bot if stopped. - """ - msg = self._rpc._rpc_start() - return jsonify(msg) - - @require_login - @rpc_catch_errors - def _stop(self): - """ - Handler for /stop. - Stops TradeThread in bot if running - """ - msg = self._rpc._rpc_stop() - return jsonify(msg) - - @require_login - @rpc_catch_errors - def _stopbuy(self): - """ - Handler for /stopbuy. - Sets max_open_trades to 0 and gracefully sells all open trades - """ - msg = self._rpc._rpc_stopbuy() - return jsonify(msg) - - @rpc_catch_errors - def _ping(self): - """ - simple ping version - """ - return jsonify({"status": "pong"}) - - @require_login - @rpc_catch_errors - def _version(self): - """ - Prints the bot's version - """ - return jsonify({"version": __version__}) - - @require_login - @rpc_catch_errors - def _show_config(self): - """ - Prints the bot's version - """ - return jsonify(RPC._rpc_show_config(self._config, self._rpc._freqtrade.state)) - - @require_login - @rpc_catch_errors - def _reload_config(self): - """ - Handler for /reload_config. - Triggers a config file reload - """ - msg = self._rpc._rpc_reload_config() - return jsonify(msg) - - @require_login - @rpc_catch_errors - def _count(self): - """ - Handler for /count. - Returns the number of trades running - """ - msg = self._rpc._rpc_count() - return jsonify(msg) - - @require_login - @rpc_catch_errors - def _locks(self): - """ - Handler for /locks. - Returns the currently active locks. - """ - return jsonify(self._rpc._rpc_locks()) - - @require_login - @rpc_catch_errors - def _daily(self): - """ - Returns the last X days trading stats summary. - - :return: stats - """ - timescale = request.args.get('timescale', 7) - timescale = int(timescale) - - stats = self._rpc._rpc_daily_profit(timescale, - self._config['stake_currency'], - self._config.get('fiat_display_currency', '') - ) - - return jsonify(stats) - - @require_login - @rpc_catch_errors - def _get_logs(self): - """ - Returns latest logs - get: - param: - limit: Only get a certain number of records - """ - limit = int(request.args.get('limit', 0)) or None - return jsonify(RPC._rpc_get_logs(limit)) - - @require_login - @rpc_catch_errors - def _edge(self): - """ - Returns information related to Edge. - :return: edge stats - """ - stats = self._rpc._rpc_edge() - - return jsonify(stats) - - @require_login - @rpc_catch_errors - def _profit(self): - """ - Handler for /profit. - - Returns a cumulative profit statistics - :return: stats - """ - - stats = self._rpc._rpc_trade_statistics(self._config['stake_currency'], - self._config.get('fiat_display_currency') - ) - - return jsonify(stats) - - @require_login - @rpc_catch_errors - def _stats(self): - """ - Handler for /stats. - Returns a Object with "durations" and "sell_reasons" as keys. - """ - - stats = self._rpc._rpc_stats() - - return jsonify(stats) - - @require_login - @rpc_catch_errors - def _performance(self): - """ - Handler for /performance. - - Returns a cumulative performance statistics - :return: stats - """ - stats = self._rpc._rpc_performance() - - return jsonify(stats) - - @require_login - @rpc_catch_errors - def _status(self): - """ - Handler for /status. - - Returns the current status of the trades in json format - """ - try: - results = self._rpc._rpc_trade_status() - return jsonify(results) - except RPCException: - return jsonify([]) - - @require_login - @rpc_catch_errors - def _balance(self): - """ - Handler for /balance. - - Returns the current status of the trades in json format - """ - results = self._rpc._rpc_balance(self._config['stake_currency'], - self._config.get('fiat_display_currency', '')) - return jsonify(results) - - @require_login - @rpc_catch_errors - def _trades(self): - """ - Handler for /trades. - - Returns the X last trades in json format - """ - limit = int(request.args.get('limit', 0)) - results = self._rpc._rpc_trade_history(limit) - return jsonify(results) - - @require_login - @rpc_catch_errors - def _trades_delete(self, tradeid: int): - """ - Handler for DELETE /trades/ endpoint. - Removes the trade from the database (tries to cancel open orders first!) - get: - param: - tradeid: Numeric trade-id assigned to the trade. - """ - result = self._rpc._rpc_delete(tradeid) - return jsonify(result) - - @require_login - @rpc_catch_errors - def _whitelist(self): - """ - Handler for /whitelist. - """ - results = self._rpc._rpc_whitelist() - return jsonify(results) - - @require_login - @rpc_catch_errors - def _blacklist(self): - """ - Handler for /blacklist. - """ - add = request.json.get("blacklist", None) if request.method == 'POST' else None - results = self._rpc._rpc_blacklist(add) - return jsonify(results) - - @require_login - @rpc_catch_errors - def _forcebuy(self): - """ - Handler for /forcebuy. - """ - asset = request.json.get("pair") - price = request.json.get("price", None) - price = float(price) if price is not None else price - - trade = self._rpc._rpc_forcebuy(asset, price) - if trade: - return jsonify(trade.to_json()) - else: - return jsonify({"status": f"Error buying pair {asset}."}) - - @require_login - @rpc_catch_errors - def _forcesell(self): - """ - Handler for /forcesell. - """ - tradeid = request.json.get("tradeid") - results = self._rpc._rpc_forcesell(tradeid) - return jsonify(results) - - @require_login - @rpc_catch_errors - def _analysed_candles(self): - """ - Handler for /pair_candles. - Returns the dataframe the bot is using during live/dry operations. - Takes the following get arguments: - get: - parameters: - - pair: Pair - - timeframe: Timeframe to get data for (should be aligned to strategy.timeframe) - - limit: Limit return length to the latest X candles - """ - pair = request.args.get("pair") - timeframe = request.args.get("timeframe") - limit = request.args.get("limit", type=int) - if not pair or not timeframe: - return self.rest_error("Mandatory parameter missing.", 400) - - results = self._rpc._rpc_analysed_dataframe(pair, timeframe, limit) - return jsonify(results) - - @require_login - @rpc_catch_errors - def _analysed_history(self): - """ - Handler for /pair_history. - Returns the dataframe of a given timerange - Takes the following get arguments: - get: - parameters: - - pair: Pair - - timeframe: Timeframe to get data for (should be aligned to strategy.timeframe) - - strategy: Strategy to use - Must exist in configured strategy-path! - - timerange: timerange in the format YYYYMMDD-YYYYMMDD (YYYYMMDD- or (-YYYYMMDD)) - are als possible. If omitted uses all available data. - """ - pair = request.args.get("pair") - timeframe = request.args.get("timeframe") - timerange = request.args.get("timerange") - strategy = request.args.get("strategy") - - if not pair or not timeframe or not timerange or not strategy: - return self.rest_error("Mandatory parameter missing.", 400) - - config = deepcopy(self._config) - config.update({ - 'strategy': strategy, - }) - results = RPC._rpc_analysed_history_full(config, pair, timeframe, timerange) - return jsonify(results) - - @require_login - @rpc_catch_errors - def _plot_config(self): - """ - Handler for /plot_config. - """ - return jsonify(self._rpc._rpc_plot_config()) - - @require_login - @rpc_catch_errors - def _list_strategies(self): - directory = Path(self._config.get( - 'strategy_path', self._config['user_data_dir'] / USERPATH_STRATEGIES)) - from freqtrade.resolvers.strategy_resolver import StrategyResolver - strategy_objs = StrategyResolver.search_all_objects(directory, False) - strategy_objs = sorted(strategy_objs, key=lambda x: x['name']) - - return jsonify({'strategies': [x['name'] for x in strategy_objs]}) - - @require_login - @rpc_catch_errors - def _get_strategy(self, strategy: str): - """ - Get a single strategy - get: - parameters: - - strategy: Only get this strategy - """ - config = deepcopy(self._config) - from freqtrade.resolvers.strategy_resolver import StrategyResolver - try: - strategy_obj = StrategyResolver._load_strategy(strategy, config, - extra_dir=config.get('strategy_path')) - except OperationalException: - return self.rest_error("Strategy not found.", 404) - - return jsonify({ - 'strategy': strategy_obj.get_strategy_name(), - 'code': strategy_obj.__source__, - }) - - @require_login - @rpc_catch_errors - def _list_available_pairs(self): - """ - Handler for /available_pairs. - Returns an object, with pairs, available pair length and pair_interval combinations - Takes the following get arguments: - get: - parameters: - - stake_currency: Filter on this stake currency - - timeframe: Timeframe to get data for Filter elements to this timeframe - """ - timeframe = request.args.get("timeframe") - stake_currency = request.args.get("stake_currency") - - from freqtrade.data.history import get_datahandler - dh = get_datahandler(self._config['datadir'], self._config.get('dataformat_ohlcv', None)) - - pair_interval = dh.ohlcv_get_available_data(self._config['datadir']) - - if timeframe: - pair_interval = [pair for pair in pair_interval if pair[1] == timeframe] - if stake_currency: - pair_interval = [pair for pair in pair_interval if pair[0].endswith(stake_currency)] - pair_interval = sorted(pair_interval, key=lambda x: x[0]) - - pairs = list({x[0] for x in pair_interval}) - - result = { - 'length': len(pairs), - 'pairs': pairs, - 'pair_interval': pair_interval, - } - return jsonify(result) diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server2/webserver.py index b3e6eb0dc..793e581ce 100644 --- a/freqtrade/rpc/api_server2/webserver.py +++ b/freqtrade/rpc/api_server2/webserver.py @@ -1,6 +1,6 @@ import logging from ipaddress import IPv4Address -from typing import Any, Dict, Optional +from typing import Any, Dict import uvicorn from fastapi import Depends, FastAPI diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 8017293b4..c49f6a1d1 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -5,23 +5,22 @@ Unit test file for rpc/api_server.py from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import ANY, MagicMock, PropertyMock -from fastapi.applications import FastAPI import pytest -from fastapi.testclient import TestClient from fastapi import FastAPI +from fastapi.testclient import TestClient from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ from freqtrade.loggers import setup_logging, setup_logging_pre from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC -from freqtrade.rpc.api_server111 import BASE_URI # , ApiServer from freqtrade.rpc.api_server2 import ApiServer from freqtrade.state import RunMode, State from tests.conftest import create_mock_trades, get_patched_freqtradebot, log_has, patch_get_signal +BASE_URI = "/api/v1" _TEST_USER = "FreqTrader" _TEST_PASS = "SuperSecurePassword1!" From 29ce323649ec354c15c4aa61bcbeb99c38b0f4a4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Dec 2020 14:21:05 +0100 Subject: [PATCH 056/183] Fix wrong hyperoptlosstest --- tests/optimize/test_hyperoptloss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py index 63012ee48..f7910e6d6 100644 --- a/tests/optimize/test_hyperoptloss.py +++ b/tests/optimize/test_hyperoptloss.py @@ -20,7 +20,7 @@ def test_hyperoptlossresolver(mocker, default_conf) -> None: hl = ShortTradeDurHyperOptLoss mocker.patch( 'freqtrade.resolvers.hyperopt_resolver.HyperOptLossResolver.load_object', - MagicMock(return_value=hl) + MagicMock(return_value=hl()) ) default_conf.update({'hyperopt_loss': 'SharpeHyperOptLossDaily'}) x = HyperOptLossResolver.load_hyperoptloss(default_conf) From 790f8336531c2ff9e22c556e6658659deec7a2eb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Dec 2020 14:57:01 +0100 Subject: [PATCH 057/183] Some more tests around api_auth --- freqtrade/rpc/api_server2/api_auth.py | 2 +- freqtrade/rpc/api_server2/uvicorn_threaded.py | 5 -- tests/rpc/test_rpc_apiserver.py | 57 +++++++++++++++++-- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_auth.py b/freqtrade/rpc/api_server2/api_auth.py index 599f6b53c..595107acb 100644 --- a/freqtrade/rpc/api_server2/api_auth.py +++ b/freqtrade/rpc/api_server2/api_auth.py @@ -73,7 +73,7 @@ def http_basic_or_jwt_token(form_data: HTTPBasicCredentials = Depends(httpbasic) raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, - detail="Incorrect username or password", + detail="Unauthorized", ) diff --git a/freqtrade/rpc/api_server2/uvicorn_threaded.py b/freqtrade/rpc/api_server2/uvicorn_threaded.py index ce7089bed..1554a8e52 100644 --- a/freqtrade/rpc/api_server2/uvicorn_threaded.py +++ b/freqtrade/rpc/api_server2/uvicorn_threaded.py @@ -19,13 +19,8 @@ class UvicornServer(uvicorn.Server): def run_in_thread(self): self.thread = threading.Thread(target=self.run) self.thread.start() - # try: while not self.started: time.sleep(1e-3) - # yield - # finally: - # self.should_exit = True - # thread.join() def cleanup(self): self.should_exit = True diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index c49f6a1d1..ee8976741 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -3,6 +3,12 @@ Unit test file for rpc/api_server.py """ from datetime import datetime, timedelta, timezone + +import uvicorn +from freqtrade.rpc.api_server2.uvicorn_threaded import UvicornServer + +from fastapi.exceptions import HTTPException +from freqtrade.rpc.api_server2.api_auth import create_token, get_user_from_token from pathlib import Path from unittest.mock import ANY, MagicMock, PropertyMock @@ -83,6 +89,26 @@ def test_api_not_found(botclient): assert rc.json() == {"detail": "Not Found"} +def test_api_auth(): + with pytest.raises(ValueError): + create_token({'sub': 'Freqtrade'}, token_type="NotATokenType") + + token = create_token({'sub': 'Freqtrade'}, ) + assert isinstance(token, bytes) + + u = get_user_from_token(token) + assert u == 'Freqtrade' + with pytest.raises(HTTPException): + get_user_from_token(token, token_type='refresh') + # Create invalid token + token = create_token({'sub`': 'Freqtrade'}, ) + with pytest.raises(HTTPException): + get_user_from_token(token) + + with pytest.raises(HTTPException): + get_user_from_token(b'not_a_token') + + def test_api_unauthorized(botclient): ftbot, client = botclient rc = client.get(f"{BASE_URI}/ping") @@ -92,31 +118,36 @@ def test_api_unauthorized(botclient): # Don't send user/pass information rc = client.get(f"{BASE_URI}/version") assert_response(rc, 401, needs_cors=False) - assert rc.json() == {'error': 'Unauthorized'} + assert rc.json() == {'detail': 'Unauthorized'} # Change only username ftbot.config['api_server']['username'] = 'Ftrader' rc = client_get(client, f"{BASE_URI}/version") assert_response(rc, 401) - assert rc.json() == {'error': 'Unauthorized'} + assert rc.json() == {'detail': 'Unauthorized'} # Change only password ftbot.config['api_server']['username'] = _TEST_USER ftbot.config['api_server']['password'] = 'WrongPassword' rc = client_get(client, f"{BASE_URI}/version") assert_response(rc, 401) - assert rc.json() == {'error': 'Unauthorized'} + assert rc.json() == {'detail': 'Unauthorized'} ftbot.config['api_server']['username'] = 'Ftrader' ftbot.config['api_server']['password'] = 'WrongPassword' rc = client_get(client, f"{BASE_URI}/version") assert_response(rc, 401) - assert rc.json() == {'error': 'Unauthorized'} + assert rc.json() == {'detail': 'Unauthorized'} def test_api_token_login(botclient): ftbot, client = botclient + rc = client.post(f"{BASE_URI}/token/login", + data=None, + headers={'Authorization': _basic_auth_str('WRONG_USER', 'WRONG_PASS'), + 'Origin': 'http://example.com'}) + assert_response(rc, 401) rc = client_post(client, f"{BASE_URI}/token/login") assert_response(rc) assert 'access_token' in rc.json() @@ -183,6 +214,24 @@ def test_api__init__(default_conf, mocker): assert apiserver._config == default_conf +def test_api_UvicornServer(default_conf, mocker): + thread_mock = mocker.patch('freqtrade.rpc.api_server2.uvicorn_threaded.threading.Thread') + s = UvicornServer(uvicorn.Config(MagicMock(), port=8080, host='127.0.0.1')) + assert thread_mock.call_count == 0 + + s.install_signal_handlers() + # Original implementation starts a thread - make sure that's not the case + assert thread_mock.call_count == 0 + + # Fake started to avoid sleeping forever + s.started = True + s.run_in_thread() + assert thread_mock.call_count == 1 + + s.cleanup() + assert s.should_exit is True + + def test_api_run(default_conf, mocker, caplog): default_conf.update({"api_server": {"enabled": True, "listen_ip_address": "127.0.0.1", From 1717121f10d10b9d123c5c2ba95e3a9828ec4d1b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Dec 2020 15:24:49 +0100 Subject: [PATCH 058/183] Properly use JWT secret key --- freqtrade/rpc/api_server2/api_auth.py | 43 ++++++++++++++------------ freqtrade/rpc/api_server2/deps.py | 4 +++ freqtrade/rpc/api_server2/webserver.py | 5 +++ tests/rpc/test_rpc_apiserver.py | 18 ++++++----- 4 files changed, 42 insertions(+), 28 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_auth.py b/freqtrade/rpc/api_server2/api_auth.py index 595107acb..00ae60ed2 100644 --- a/freqtrade/rpc/api_server2/api_auth.py +++ b/freqtrade/rpc/api_server2/api_auth.py @@ -8,34 +8,32 @@ from fastapi.security.http import HTTPBasic, HTTPBasicCredentials from freqtrade.rpc.api_server2.api_models import AccessAndRefreshToken, AccessToken -from .deps import get_config +from .deps import get_api_config -SECRET_KEY = "09d25e094faa6ca2556c818166b7a9563b93f7099f6f0f4caa6cf63b88e8d3e7" ALGORITHM = "HS256" -ACCESS_TOKEN_EXPIRE_MINUTES = 30 router_login = APIRouter() -def verify_auth(config, username: str, password: str): +def verify_auth(api_config, username: str, password: str): """Verify username/password""" - return (secrets.compare_digest(username, config['api_server'].get('username')) and - secrets.compare_digest(password, config['api_server'].get('password'))) + return (secrets.compare_digest(username, api_config.get('username')) and + secrets.compare_digest(password, api_config.get('password'))) httpbasic = HTTPBasic(auto_error=False) oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token", auto_error=False) -def get_user_from_token(token, token_type: str = "access"): +def get_user_from_token(token, secret_key: str, token_type: str = "access"): credentials_exception = HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Could not validate credentials", headers={"WWW-Authenticate": "Bearer"}, ) try: - payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) + payload = jwt.decode(token, secret_key, algorithms=[ALGORITHM]) username: str = payload.get("sub") if username is None: raise credentials_exception @@ -47,7 +45,7 @@ def get_user_from_token(token, token_type: str = "access"): return username -def create_token(data: dict, token_type: str = "access") -> str: +def create_token(data: dict, secret_key: str, token_type: str = "access") -> str: to_encode = data.copy() if token_type == "access": expire = datetime.utcnow() + timedelta(minutes=15) @@ -60,15 +58,16 @@ def create_token(data: dict, token_type: str = "access") -> str: "iat": datetime.utcnow(), "type": token_type, }) - encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM) + encoded_jwt = jwt.encode(to_encode, secret_key, algorithm=ALGORITHM) return encoded_jwt def http_basic_or_jwt_token(form_data: HTTPBasicCredentials = Depends(httpbasic), - token: str = Depends(oauth2_scheme), config=Depends(get_config)): + token: str = Depends(oauth2_scheme), + api_config=Depends(get_api_config)): if token: - return get_user_from_token(token) - elif form_data and verify_auth(config, form_data.username, form_data.password): + return get_user_from_token(token, api_config.get('jwt_secret_key', 'super-secret')) + elif form_data and verify_auth(api_config, form_data.username, form_data.password): return form_data.username raise HTTPException( @@ -78,12 +77,14 @@ def http_basic_or_jwt_token(form_data: HTTPBasicCredentials = Depends(httpbasic) @router_login.post('/token/login', response_model=AccessAndRefreshToken) -def token_login(form_data: HTTPBasicCredentials = Depends(HTTPBasic()), config=Depends(get_config)): +def token_login(form_data: HTTPBasicCredentials = Depends(HTTPBasic()), + api_config=Depends(get_api_config)): - if verify_auth(config, form_data.username, form_data.password): + if verify_auth(api_config, form_data.username, form_data.password): token_data = {'sub': form_data.username} - access_token = create_token(token_data) - refresh_token = create_token(token_data, token_type="refresh") + access_token = create_token(token_data, api_config.get('jwt_secret_key', 'super-secret')) + refresh_token = create_token(token_data, api_config.get('jwt_secret_key', 'super-secret'), + token_type="refresh") return { "access_token": access_token, "refresh_token": refresh_token, @@ -97,9 +98,11 @@ def token_login(form_data: HTTPBasicCredentials = Depends(HTTPBasic()), config=D @router_login.post('/token/refresh', response_model=AccessToken) -def token_refresh(token: str = Depends(oauth2_scheme)): +def token_refresh(token: str = Depends(oauth2_scheme), api_config=Depends(get_api_config)): # Refresh token - u = get_user_from_token(token, 'refresh') + u = get_user_from_token(token, api_config.get( + 'jwt_secret_key', 'super-secret'), 'refresh') token_data = {'sub': u} - access_token = create_token(token_data, token_type="access") + access_token = create_token(token_data, api_config.get('jwt_secret_key', 'super-secret'), + token_type="access") return {'access_token': access_token} diff --git a/freqtrade/rpc/api_server2/deps.py b/freqtrade/rpc/api_server2/deps.py index 60cc6b8fb..691759012 100644 --- a/freqtrade/rpc/api_server2/deps.py +++ b/freqtrade/rpc/api_server2/deps.py @@ -7,3 +7,7 @@ def get_rpc(): def get_config(): return ApiServer._config + + +def get_api_config(): + return ApiServer._config['api_server'] diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server2/webserver.py index 793e581ce..3956e52db 100644 --- a/freqtrade/rpc/api_server2/webserver.py +++ b/freqtrade/rpc/api_server2/webserver.py @@ -86,6 +86,11 @@ class ApiServer(RPCHandler): logger.warning("SECURITY WARNING - No password for local REST Server defined. " "Please make sure that this is intentional!") + if (self._config['api_server'].get('jwt_secret_key', 'super-secret') + in ('super-secret, somethingrandom')): + logger.warning("SECURITY WARNING - `jwt_secret_key` seems to be default." + "Others may be able to log into your bot.") + logger.info('Starting Local Rest Server.') uvconfig = uvicorn.Config(self.app, port=rest_port, diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index ee8976741..95789a85f 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -23,7 +23,7 @@ from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC from freqtrade.rpc.api_server2 import ApiServer from freqtrade.state import RunMode, State -from tests.conftest import create_mock_trades, get_patched_freqtradebot, log_has, patch_get_signal +from tests.conftest import create_mock_trades, get_patched_freqtradebot, log_has, log_has_re, patch_get_signal BASE_URI = "/api/v1" @@ -91,22 +91,22 @@ def test_api_not_found(botclient): def test_api_auth(): with pytest.raises(ValueError): - create_token({'sub': 'Freqtrade'}, token_type="NotATokenType") + create_token({'sub': 'Freqtrade'}, 'secret1234', token_type="NotATokenType") - token = create_token({'sub': 'Freqtrade'}, ) + token = create_token({'sub': 'Freqtrade'}, 'secret1234') assert isinstance(token, bytes) - u = get_user_from_token(token) + u = get_user_from_token(token, 'secret1234') assert u == 'Freqtrade' with pytest.raises(HTTPException): - get_user_from_token(token, token_type='refresh') + get_user_from_token(token, 'secret1234', token_type='refresh') # Create invalid token - token = create_token({'sub`': 'Freqtrade'}, ) + token = create_token({'sub`': 'Freqtrade'}, 'secret1234') with pytest.raises(HTTPException): - get_user_from_token(token) + get_user_from_token(token, 'secret1234') with pytest.raises(HTTPException): - get_user_from_token(b'not_a_token') + get_user_from_token(b'not_a_token', 'secret1234') def test_api_unauthorized(botclient): @@ -279,6 +279,8 @@ def test_api_run(default_conf, mocker, caplog): "e.g 127.0.0.1 in config.json", caplog) assert log_has("SECURITY WARNING - No password for local REST Server defined. " "Please make sure that this is intentional!", caplog) + assert log_has_re("SECURITY WARNING - `jwt_secret_key` seems to be default.*", caplog) + # Test crashing flask caplog.clear() From 68d148e72d1b5da8951d976c6ce96e355ac0165d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 27 Dec 2020 15:54:05 +0100 Subject: [PATCH 059/183] Allow configuration of openAPI interface --- config_full.json.example | 1 + docs/rest-api.md | 6 ++++++ freqtrade/rpc/api_server2/webserver.py | 10 ++++++++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index e69e52469..ef9f45363 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -165,6 +165,7 @@ "listen_ip_address": "127.0.0.1", "listen_port": 8080, "verbosity": "info", + "enable_openapi": false, "jwt_secret_key": "somethingrandom", "CORS_origins": [], "username": "freqtrader", diff --git a/docs/rest-api.md b/docs/rest-api.md index 9bb35ce91..279373c50 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -12,6 +12,7 @@ Sample configuration: "listen_ip_address": "127.0.0.1", "listen_port": 8080, "verbosity": "info", + "enable_openapi": false, "jwt_secret_key": "somethingrandom", "CORS_origins": [], "username": "Freqtrader", @@ -263,6 +264,11 @@ whitelist ``` +## OpenAPI interface + +To enable the builtin openAPI interface, specify `"enable_openapi": true` in the api_server configuration. +This will enable the Swagger UI at the `/docs` endpoint. By default, that's running at http://localhost:8080/docs/ - but it'll depend on your settings. + ## Advanced API usage using JWT tokens !!! Note diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server2/webserver.py index 3956e52db..c5cc30156 100644 --- a/freqtrade/rpc/api_server2/webserver.py +++ b/freqtrade/rpc/api_server2/webserver.py @@ -26,8 +26,13 @@ class ApiServer(RPCHandler): ApiServer._rpc = rpc ApiServer._config = config + api_config = self._config['api_server'] - self.app = FastAPI(title="Freqtrade API") + self.app = FastAPI(title="Freqtrade API", + openapi_url='openapi.json' if api_config.get( + 'enable_openapi') else None, + redoc_url=None, + ) self.configure_app(self.app, self._config) self.start_api() @@ -92,10 +97,11 @@ class ApiServer(RPCHandler): "Others may be able to log into your bot.") logger.info('Starting Local Rest Server.') + verbosity = self._config['api_server'].get('verbosity', 'info') uvconfig = uvicorn.Config(self.app, port=rest_port, host=rest_ip, - access_log=True) + access_log=True if verbosity != 'error' else False) try: self._server = UvicornServer(uvconfig) self._server.run_in_thread() From 346542e5cddc05892521c1fc7d01223126efc5f5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 29 Dec 2020 17:23:28 +0100 Subject: [PATCH 060/183] Remove flask dependency --- freqtrade/rpc/api_server2/api_auth.py | 1 - freqtrade/rpc/api_server2/webserver.py | 3 ++- requirements.txt | 5 ----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_auth.py b/freqtrade/rpc/api_server2/api_auth.py index 00ae60ed2..6f5d051d3 100644 --- a/freqtrade/rpc/api_server2/api_auth.py +++ b/freqtrade/rpc/api_server2/api_auth.py @@ -93,7 +93,6 @@ def token_login(form_data: HTTPBasicCredentials = Depends(HTTPBasic()), raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Incorrect username or password", - headers={"WWW-Authenticate": "Basic"}, ) diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server2/webserver.py index c5cc30156..f603ab160 100644 --- a/freqtrade/rpc/api_server2/webserver.py +++ b/freqtrade/rpc/api_server2/webserver.py @@ -101,7 +101,8 @@ class ApiServer(RPCHandler): uvconfig = uvicorn.Config(self.app, port=rest_port, host=rest_ip, - access_log=True if verbosity != 'error' else False) + access_log=True if verbosity != 'error' else False, + ) try: self._server = UvicornServer(uvconfig) self._server.run_in_thread() diff --git a/requirements.txt b/requirements.txt index 4b439079b..779cc3771 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,11 +27,6 @@ python-rapidjson==1.0 # Notify systemd sdnotify==0.3.2 -# Api server -flask==1.1.2 -flask-jwt-extended==3.25.0 -flask-cors==3.0.9 - # API Server fastapi==0.63.0 uvicorn==0.13.2 From eb20f6e7d06b8de23c0a22d31e11bd60ace93c94 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Dec 2020 10:22:28 +0100 Subject: [PATCH 061/183] Align auth token to flask version to prevent user-logout --- freqtrade/rpc/api_server2/api_auth.py | 6 +++--- freqtrade/rpc/rpc_manager.py | 2 -- tests/rpc/test_rpc_apiserver.py | 6 +++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/freqtrade/rpc/api_server2/api_auth.py b/freqtrade/rpc/api_server2/api_auth.py index 6f5d051d3..a02accb18 100644 --- a/freqtrade/rpc/api_server2/api_auth.py +++ b/freqtrade/rpc/api_server2/api_auth.py @@ -34,7 +34,7 @@ def get_user_from_token(token, secret_key: str, token_type: str = "access"): ) try: payload = jwt.decode(token, secret_key, algorithms=[ALGORITHM]) - username: str = payload.get("sub") + username: str = payload.get("identity", {}).get('u') if username is None: raise credentials_exception if payload.get("type") != token_type: @@ -81,7 +81,7 @@ def token_login(form_data: HTTPBasicCredentials = Depends(HTTPBasic()), api_config=Depends(get_api_config)): if verify_auth(api_config, form_data.username, form_data.password): - token_data = {'sub': form_data.username} + token_data = {'identity': {'u': form_data.username}} access_token = create_token(token_data, api_config.get('jwt_secret_key', 'super-secret')) refresh_token = create_token(token_data, api_config.get('jwt_secret_key', 'super-secret'), token_type="refresh") @@ -101,7 +101,7 @@ def token_refresh(token: str = Depends(oauth2_scheme), api_config=Depends(get_ap # Refresh token u = get_user_from_token(token, api_config.get( 'jwt_secret_key', 'super-secret'), 'refresh') - token_data = {'sub': u} + token_data = {'identity': {'u': u}} access_token = create_token(token_data, api_config.get('jwt_secret_key', 'super-secret'), token_type="access") return {'access_token': access_token} diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 2afd39eda..369dfa5c9 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -34,8 +34,6 @@ class RPCManager: # Enable local rest api server for cmd line control if config.get('api_server', {}).get('enabled', False): logger.info('Enabling rpc.api_server') - # from freqtrade.rpc.api_server import ApiServer - # TODO: Remove the above import from freqtrade.rpc.api_server2 import ApiServer self.registered_modules.append(ApiServer(self._rpc, config)) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 95789a85f..470032357 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -91,9 +91,9 @@ def test_api_not_found(botclient): def test_api_auth(): with pytest.raises(ValueError): - create_token({'sub': 'Freqtrade'}, 'secret1234', token_type="NotATokenType") + create_token({'identity': {'u': 'Freqtrade'}}, 'secret1234', token_type="NotATokenType") - token = create_token({'sub': 'Freqtrade'}, 'secret1234') + token = create_token({'identity': {'u': 'Freqtrade'}}, 'secret1234') assert isinstance(token, bytes) u = get_user_from_token(token, 'secret1234') @@ -101,7 +101,7 @@ def test_api_auth(): with pytest.raises(HTTPException): get_user_from_token(token, 'secret1234', token_type='refresh') # Create invalid token - token = create_token({'sub`': 'Freqtrade'}, 'secret1234') + token = create_token({'identity': {'u1': 'Freqrade'}}, 'secret1234') with pytest.raises(HTTPException): get_user_from_token(token, 'secret1234') From b2ab553a31d99338fa5add94bb4be2b53997fc18 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 31 Dec 2020 11:01:50 +0100 Subject: [PATCH 062/183] Rename api_server2 module to apiserver --- .../{api_server2 => api_server}/__init__.py | 0 .../{api_server2 => api_server}/api_auth.py | 7 ++--- .../{api_server2 => api_server}/api_models.py | 2 +- .../rpc/{api_server2 => api_server}/api_v1.py | 6 ++-- .../rpc/{api_server2 => api_server}/deps.py | 0 .../uvicorn_threaded.py | 0 .../{api_server2 => api_server}/webserver.py | 2 +- freqtrade/rpc/rpc_manager.py | 2 +- tests/rpc/test_rpc_apiserver.py | 29 +++++++++---------- tests/rpc/test_rpc_manager.py | 4 +-- 10 files changed, 25 insertions(+), 27 deletions(-) rename freqtrade/rpc/{api_server2 => api_server}/__init__.py (100%) rename freqtrade/rpc/{api_server2 => api_server}/api_auth.py (95%) rename freqtrade/rpc/{api_server2 => api_server}/api_models.py (98%) rename freqtrade/rpc/{api_server2 => api_server}/api_v1.py (97%) rename freqtrade/rpc/{api_server2 => api_server}/deps.py (100%) rename freqtrade/rpc/{api_server2 => api_server}/uvicorn_threaded.py (100%) rename freqtrade/rpc/{api_server2 => api_server}/webserver.py (99%) diff --git a/freqtrade/rpc/api_server2/__init__.py b/freqtrade/rpc/api_server/__init__.py similarity index 100% rename from freqtrade/rpc/api_server2/__init__.py rename to freqtrade/rpc/api_server/__init__.py diff --git a/freqtrade/rpc/api_server2/api_auth.py b/freqtrade/rpc/api_server/api_auth.py similarity index 95% rename from freqtrade/rpc/api_server2/api_auth.py rename to freqtrade/rpc/api_server/api_auth.py index a02accb18..8d1316906 100644 --- a/freqtrade/rpc/api_server2/api_auth.py +++ b/freqtrade/rpc/api_server/api_auth.py @@ -6,9 +6,8 @@ from fastapi import APIRouter, Depends, HTTPException, status from fastapi.security import OAuth2PasswordBearer from fastapi.security.http import HTTPBasic, HTTPBasicCredentials -from freqtrade.rpc.api_server2.api_models import AccessAndRefreshToken, AccessToken - -from .deps import get_api_config +from freqtrade.rpc.api_server.api_models import AccessAndRefreshToken, AccessToken +from freqtrade.rpc.api_server.deps import get_api_config ALGORITHM = "HS256" @@ -45,7 +44,7 @@ def get_user_from_token(token, secret_key: str, token_type: str = "access"): return username -def create_token(data: dict, secret_key: str, token_type: str = "access") -> str: +def create_token(data: dict, secret_key: str, token_type: str = "access") -> bytes: to_encode = data.copy() if token_type == "access": expire = datetime.utcnow() + timedelta(minutes=15) diff --git a/freqtrade/rpc/api_server2/api_models.py b/freqtrade/rpc/api_server/api_models.py similarity index 98% rename from freqtrade/rpc/api_server2/api_models.py rename to freqtrade/rpc/api_server/api_models.py index c9e4ee5cc..a8b03eac5 100644 --- a/freqtrade/rpc/api_server2/api_models.py +++ b/freqtrade/rpc/api_server/api_models.py @@ -167,7 +167,7 @@ class DeleteTrade(BaseModel): class PlotConfig(BaseModel): - main_plot: Optional[Dict[str, Any]] + main_plot: Dict[str, Any] subplots: Optional[Dict[str, Any]] diff --git a/freqtrade/rpc/api_server2/api_v1.py b/freqtrade/rpc/api_server/api_v1.py similarity index 97% rename from freqtrade/rpc/api_server2/api_v1.py rename to freqtrade/rpc/api_server/api_v1.py index 21c525850..af9592a7b 100644 --- a/freqtrade/rpc/api_server2/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -83,7 +83,7 @@ def status(rpc: RPC = Depends(get_rpc)): # TODO: Missing response model @router.get('/trades', tags=['info', 'trading']) -def trades(limit: Optional[int] = 0, rpc: RPC = Depends(get_rpc)): +def trades(limit: int = 0, rpc: RPC = Depends(get_rpc)): return rpc._rpc_trade_history(limit) @@ -180,8 +180,8 @@ def pair_history(pair: str, timeframe: str, timerange: str, strategy: str, return RPC._rpc_analysed_history_full(config, pair, timeframe, timerange) -@router.get('/plot_config', response_model=Union[Dict, PlotConfig], tags=['candle data']) -def plot_config(rpc=Depends(get_rpc)): +@router.get('/plot_config', response_model=Union[PlotConfig, Dict], tags=['candle data']) +def plot_config(rpc: RPC = Depends(get_rpc)): return rpc._rpc_plot_config() diff --git a/freqtrade/rpc/api_server2/deps.py b/freqtrade/rpc/api_server/deps.py similarity index 100% rename from freqtrade/rpc/api_server2/deps.py rename to freqtrade/rpc/api_server/deps.py diff --git a/freqtrade/rpc/api_server2/uvicorn_threaded.py b/freqtrade/rpc/api_server/uvicorn_threaded.py similarity index 100% rename from freqtrade/rpc/api_server2/uvicorn_threaded.py rename to freqtrade/rpc/api_server/uvicorn_threaded.py diff --git a/freqtrade/rpc/api_server2/webserver.py b/freqtrade/rpc/api_server/webserver.py similarity index 99% rename from freqtrade/rpc/api_server2/webserver.py rename to freqtrade/rpc/api_server/webserver.py index f603ab160..caddcba84 100644 --- a/freqtrade/rpc/api_server2/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) class ApiServer(RPCHandler): - _rpc: RPC = None + _rpc: RPC _config: Dict[str, Any] = {} def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 369dfa5c9..7977d68de 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -34,7 +34,7 @@ class RPCManager: # Enable local rest api server for cmd line control if config.get('api_server', {}).get('enabled', False): logger.info('Enabling rpc.api_server') - from freqtrade.rpc.api_server2 import ApiServer + from freqtrade.rpc.api_server import ApiServer self.registered_modules.append(ApiServer(self._rpc, config)) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 470032357..8cad9d808 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -3,17 +3,13 @@ Unit test file for rpc/api_server.py """ from datetime import datetime, timedelta, timezone - -import uvicorn -from freqtrade.rpc.api_server2.uvicorn_threaded import UvicornServer - -from fastapi.exceptions import HTTPException -from freqtrade.rpc.api_server2.api_auth import create_token, get_user_from_token from pathlib import Path from unittest.mock import ANY, MagicMock, PropertyMock import pytest +import uvicorn from fastapi import FastAPI +from fastapi.exceptions import HTTPException from fastapi.testclient import TestClient from requests.auth import _basic_auth_str @@ -21,9 +17,12 @@ from freqtrade.__init__ import __version__ from freqtrade.loggers import setup_logging, setup_logging_pre from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC -from freqtrade.rpc.api_server2 import ApiServer +from freqtrade.rpc.api_server import ApiServer +from freqtrade.rpc.api_server.api_auth import create_token, get_user_from_token +from freqtrade.rpc.api_server.uvicorn_threaded import UvicornServer from freqtrade.state import RunMode, State -from tests.conftest import create_mock_trades, get_patched_freqtradebot, log_has, log_has_re, patch_get_signal +from tests.conftest import (create_mock_trades, get_patched_freqtradebot, log_has, log_has_re, + patch_get_signal) BASE_URI = "/api/v1" @@ -46,7 +45,7 @@ def botclient(default_conf, mocker): ftbot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(ftbot) - mocker.patch('freqtrade.rpc.api_server2.ApiServer.start_api', MagicMock()) + mocker.patch('freqtrade.rpc.api_server.ApiServer.start_api', MagicMock()) apiserver = ApiServer(rpc, default_conf) yield ftbot, TestClient(apiserver.app) # Cleanup ... ? @@ -209,13 +208,13 @@ def test_api__init__(default_conf, mocker): "password": "testPass", }}) mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) - mocker.patch('freqtrade.rpc.api_server2.webserver.ApiServer.start_api', MagicMock()) + mocker.patch('freqtrade.rpc.api_server.webserver.ApiServer.start_api', MagicMock()) apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) assert apiserver._config == default_conf def test_api_UvicornServer(default_conf, mocker): - thread_mock = mocker.patch('freqtrade.rpc.api_server2.uvicorn_threaded.threading.Thread') + thread_mock = mocker.patch('freqtrade.rpc.api_server.uvicorn_threaded.threading.Thread') s = UvicornServer(uvicorn.Config(MagicMock(), port=8080, host='127.0.0.1')) assert thread_mock.call_count == 0 @@ -242,7 +241,7 @@ def test_api_run(default_conf, mocker, caplog): mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) server_mock = MagicMock() - mocker.patch('freqtrade.rpc.api_server2.webserver.UvicornServer', server_mock) + mocker.patch('freqtrade.rpc.api_server.webserver.UvicornServer', server_mock) apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) @@ -281,10 +280,10 @@ def test_api_run(default_conf, mocker, caplog): "Please make sure that this is intentional!", caplog) assert log_has_re("SECURITY WARNING - `jwt_secret_key` seems to be default.*", caplog) - # Test crashing flask caplog.clear() - mocker.patch('freqtrade.rpc.api_server2.webserver.UvicornServer', MagicMock(side_effect=Exception)) + mocker.patch('freqtrade.rpc.api_server.webserver.UvicornServer', + MagicMock(side_effect=Exception)) apiserver.start_api() assert log_has("Api server failed to start.", caplog) @@ -300,7 +299,7 @@ def test_api_cleanup(default_conf, mocker, caplog): server_mock = MagicMock() server_mock.cleanup = MagicMock() - mocker.patch('freqtrade.rpc.api_server2.webserver.UvicornServer', server_mock) + mocker.patch('freqtrade.rpc.api_server.webserver.UvicornServer', server_mock) apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index e63d629b8..3068e9764 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -160,7 +160,7 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: def test_init_apiserver_disabled(mocker, default_conf, caplog) -> None: caplog.set_level(logging.DEBUG) run_mock = MagicMock() - mocker.patch('freqtrade.rpc.api_server2.ApiServer.start_api', run_mock) + mocker.patch('freqtrade.rpc.api_server.ApiServer.start_api', run_mock) default_conf['telegram']['enabled'] = False rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) @@ -172,7 +172,7 @@ def test_init_apiserver_disabled(mocker, default_conf, caplog) -> None: def test_init_apiserver_enabled(mocker, default_conf, caplog) -> None: caplog.set_level(logging.DEBUG) run_mock = MagicMock() - mocker.patch('freqtrade.rpc.api_server2.ApiServer.start_api', run_mock) + mocker.patch('freqtrade.rpc.api_server.ApiServer.start_api', run_mock) default_conf["telegram"]["enabled"] = False default_conf["api_server"] = {"enabled": True, From 718f2b24d2d8f70c1cd1ac9b92282501faed2032 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Jan 2021 19:13:32 +0100 Subject: [PATCH 063/183] Don't use relative imports --- freqtrade/rpc/api_server/api_v1.py | 15 ++++++++------- freqtrade/rpc/api_server/webserver.py | 9 ++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index af9592a7b..3ab378d15 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -10,15 +10,16 @@ from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.data.history import get_datahandler from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC +from freqtrade.rpc.api_server.api_models import (AvailablePairs, Balances, BlacklistPayload, + BlacklistResponse, Count, Daily, DeleteTrade, + ForceBuyPayload, ForceSellPayload, Locks, Logs, + PairHistory, PerformanceEntry, Ping, PlotConfig, + Profit, ResultMsg, Stats, StatusMsg, + StrategyListResponse, StrategyResponse, Version, + WhitelistResponse) +from freqtrade.rpc.api_server.deps import get_config, get_rpc from freqtrade.rpc.rpc import RPCException -from .api_models import (AvailablePairs, Balances, BlacklistPayload, BlacklistResponse, Count, - Daily, DeleteTrade, ForceBuyPayload, ForceSellPayload, Locks, Logs, - PairHistory, PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, Stats, - StatusMsg, StrategyListResponse, StrategyResponse, Version, - WhitelistResponse) -from .deps import get_config, get_rpc - # Public API, requires no auth. router_public = APIRouter() diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index caddcba84..ac7a35a9e 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -7,10 +7,9 @@ from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware from starlette.responses import JSONResponse +from freqtrade.rpc.api_server.uvicorn_threaded import UvicornServer from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler -from .uvicorn_threaded import UvicornServer - logger = logging.getLogger(__name__) @@ -54,9 +53,9 @@ class ApiServer(RPCHandler): ) def configure_app(self, app: FastAPI, config): - from .api_auth import http_basic_or_jwt_token, router_login - from .api_v1 import router as api_v1 - from .api_v1 import router_public as api_v1_public + from freqtrade.rpc.api_server.api_auth import http_basic_or_jwt_token, router_login + from freqtrade.rpc.api_server.api_v1 import router as api_v1 + from freqtrade.rpc.api_server.api_v1 import router_public as api_v1_public app.include_router(api_v1_public, prefix="/api/v1") app.include_router(api_v1, prefix="/api/v1", From 29f4dd1dcd61f7bc6471ab83a262d9b36b1a8167 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Jan 2021 19:36:03 +0100 Subject: [PATCH 064/183] Enhance some response models --- freqtrade/rpc/api_server/api_models.py | 54 ++++++++++++++++++++++++++ freqtrade/rpc/api_server/api_v1.py | 10 ++--- tests/rpc/test_rpc_apiserver.py | 9 +++-- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/freqtrade/rpc/api_server/api_models.py b/freqtrade/rpc/api_server/api_models.py index a8b03eac5..f4e62acdc 100644 --- a/freqtrade/rpc/api_server/api_models.py +++ b/freqtrade/rpc/api_server/api_models.py @@ -112,6 +112,60 @@ class Daily(BaseModel): stake_currency: str +class TradeSchema(BaseModel): + trade_id: str + pair: str + is_open: bool + exchange: str + amount: float + amount_requested: float + stake_amount: float + strategy: str + timeframe: str + fee_open: Optional[float] + fee_open_cost: Optional[float] + fee_open_currency: Optional[str] + fee_close: Optional[float] + fee_close_cost: Optional[float] + fee_close_currency: Optional[str] + open_date_hum: str + open_date: str + open_timestamp: int + open_rate: float + open_rate_requested: Optional[float] + open_trade_value: float + close_date_hum: Optional[str] + close_date: Optional[str] + close_timestamp: Optional[int] + close_rate: Optional[float] + close_rate_requested: Optional[float] + close_profit: Optional[float] + close_profit_pct: Optional[float] + close_profit_abs: Optional[float] + profit_ratio: Optional[float] + profit_pct: Optional[float] + profit_abs: Optional[float] + sell_reason: Optional[str] + sell_order_status: Optional[str] + stop_loss_abs: Optional[float] + stop_loss_ratio: Optional[float] + stop_loss_pct: Optional[float] + stoploss_order_id: Optional[str] + stoploss_last_update: Optional[str] + stoploss_last_update_timestamp: Optional[int] + initial_stop_loss_abs: Optional[float] + initial_stop_loss_ratio: Optional[float] + initial_stop_loss_pct: Optional[float] + min_rate: Optional[float] + max_rate: Optional[float] + open_order_id: Optional[str] + + +class TradeResponse(BaseModel): + trades: List[TradeSchema] + trades_count: int + + class LockModel(BaseModel): active: bool lock_end_time: str diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 3ab378d15..55bb9320c 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -15,7 +15,7 @@ from freqtrade.rpc.api_server.api_models import (AvailablePairs, Balances, Black ForceBuyPayload, ForceSellPayload, Locks, Logs, PairHistory, PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, Stats, StatusMsg, - StrategyListResponse, StrategyResponse, Version, + StrategyListResponse, StrategyResponse, TradeResponse, TradeSchema, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_rpc from freqtrade.rpc.rpc import RPCException @@ -74,7 +74,7 @@ def daily(timescale: int = 7, rpc: RPC = Depends(get_rpc), config=Depends(get_co # TODO: Missing response model -@router.get('/status', tags=['info']) +@router.get('/status', response_model=List[TradeSchema], tags=['info']) def status(rpc: RPC = Depends(get_rpc)): try: return rpc._rpc_trade_status() @@ -82,8 +82,7 @@ def status(rpc: RPC = Depends(get_rpc)): return [] -# TODO: Missing response model -@router.get('/trades', tags=['info', 'trading']) +@router.get('/trades', response_model=TradeResponse, tags=['info', 'trading']) def trades(limit: int = 0, rpc: RPC = Depends(get_rpc)): return rpc._rpc_trade_history(limit) @@ -105,8 +104,7 @@ def show_config(rpc: RPC = Depends(get_rpc), config=Depends(get_config)): return RPC._rpc_show_config(config, rpc._freqtrade.state) -# TODO: Missing response model -@router.post('/forcebuy', tags=['trading']) +@router.post('/forcebuy', response_model=Union[TradeSchema, StatusMsg], tags=['trading']) def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)): trade = rpc._rpc_forcebuy(payload.pair, payload.price) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 8cad9d808..5e972c694 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -853,6 +853,9 @@ def test_api_forcebuy(botclient, mocker, fee): fee_close=fee.return_value, fee_open=fee.return_value, close_rate=0.265441, + id='22', + timeframe="5m", + strategy="DefaultStrategy" )) mocker.patch("freqtrade.rpc.RPC._rpc_forcebuy", fbuy_mock) @@ -862,7 +865,7 @@ def test_api_forcebuy(botclient, mocker, fee): assert rc.json() == { 'amount': 1, 'amount_requested': 1, - 'trade_id': None, + 'trade_id': '22', 'close_date': None, 'close_date_hum': None, 'close_timestamp': None, @@ -903,8 +906,8 @@ def test_api_forcebuy(botclient, mocker, fee): 'open_trade_value': 0.24605460, 'sell_reason': None, 'sell_order_status': None, - 'strategy': None, - 'timeframe': None, + 'strategy': 'DefaultStrategy', + 'timeframe': '5m', 'exchange': 'bittrex', } From 84ced92002f38f29a4e55de1088c85e08a94b4ac Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Jan 2021 19:38:28 +0100 Subject: [PATCH 065/183] Fix mock-tests missing some fields --- freqtrade/rpc/api_server/api_models.py | 18 ++++++++++++++++-- freqtrade/rpc/api_server/api_v1.py | 10 +++++----- tests/conftest_trades.py | 6 ++++++ tests/data/test_btanalysis.py | 2 +- tests/rpc/test_rpc_apiserver.py | 8 ++++---- 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/freqtrade/rpc/api_server/api_models.py b/freqtrade/rpc/api_server/api_models.py index f4e62acdc..8c2a25ceb 100644 --- a/freqtrade/rpc/api_server/api_models.py +++ b/freqtrade/rpc/api_server/api_models.py @@ -113,7 +113,7 @@ class Daily(BaseModel): class TradeSchema(BaseModel): - trade_id: str + trade_id: int pair: str is_open: bool exchange: str @@ -121,7 +121,7 @@ class TradeSchema(BaseModel): amount_requested: float stake_amount: float strategy: str - timeframe: str + timeframe: int fee_open: Optional[float] fee_open_cost: Optional[float] fee_open_currency: Optional[str] @@ -161,6 +161,20 @@ class TradeSchema(BaseModel): open_order_id: Optional[str] +class OpenTradeSchema(TradeSchema): + stoploss_current_dist: Optional[float] + stoploss_current_dist_pct: Optional[float] + stoploss_current_dist_ratio: Optional[float] + stoploss_entry_dist: Optional[float] + stoploss_entry_dist_ratio: Optional[float] + base_currency: str + current_profit: float + current_profit_abs: float + current_profit_pct: float + current_rate: float + open_order: Optional[str] + + class TradeResponse(BaseModel): trades: List[TradeSchema] trades_count: int diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 55bb9320c..1a067faf4 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -13,9 +13,10 @@ from freqtrade.rpc import RPC from freqtrade.rpc.api_server.api_models import (AvailablePairs, Balances, BlacklistPayload, BlacklistResponse, Count, Daily, DeleteTrade, ForceBuyPayload, ForceSellPayload, Locks, Logs, - PairHistory, PerformanceEntry, Ping, PlotConfig, - Profit, ResultMsg, Stats, StatusMsg, - StrategyListResponse, StrategyResponse, TradeResponse, TradeSchema, Version, + OpenTradeSchema, PairHistory, PerformanceEntry, + Ping, PlotConfig, Profit, ResultMsg, Stats, + StatusMsg, StrategyListResponse, StrategyResponse, + TradeResponse, TradeSchema, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_rpc from freqtrade.rpc.rpc import RPCException @@ -73,8 +74,7 @@ def daily(timescale: int = 7, rpc: RPC = Depends(get_rpc), config=Depends(get_co config.get('fiat_display_currency', '')) -# TODO: Missing response model -@router.get('/status', response_model=List[TradeSchema], tags=['info']) +@router.get('/status', response_model=List[OpenTradeSchema], tags=['info']) def status(rpc: RPC = Depends(get_rpc)): try: return rpc._rpc_trade_status() diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index e84722041..fa9910b8d 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -32,6 +32,7 @@ def mock_trade_1(fee): exchange='bittrex', open_order_id='dry_run_buy_12345', strategy='DefaultStrategy', + timeframe=5, ) o = Order.parse_from_ccxt_object(mock_order_1(), 'ETH/BTC', 'buy') trade.orders.append(o) @@ -84,6 +85,7 @@ def mock_trade_2(fee): is_open=False, open_order_id='dry_run_sell_12345', strategy='DefaultStrategy', + timeframe=5, sell_reason='sell_signal', open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), close_date=datetime.now(tz=timezone.utc), @@ -132,6 +134,7 @@ def mock_trade_3(fee): pair='XRP/BTC', stake_amount=0.001, amount=123.0, + amount_requested=123.0, fee_open=fee.return_value, fee_close=fee.return_value, open_rate=0.05, @@ -139,6 +142,8 @@ def mock_trade_3(fee): close_profit=0.01, exchange='bittrex', is_open=False, + strategy='DefaultStrategy', + timeframe=5, sell_reason='roi', open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), close_date=datetime.now(tz=timezone.utc), @@ -179,6 +184,7 @@ def mock_trade_4(fee): exchange='bittrex', open_order_id='prod_buy_12345', strategy='DefaultStrategy', + timeframe=5, ) o = Order.parse_from_ccxt_object(mock_order_4(), 'ETC/BTC', 'buy') trade.orders.append(o) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 1592fac10..cdd5c08d2 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -128,7 +128,7 @@ def test_load_trades_from_db(default_conf, fee, mocker): if col not in ['index', 'open_at_end']: assert col in trades.columns trades = load_trades_from_db(db_url=default_conf['db_url'], strategy='DefaultStrategy') - assert len(trades) == 3 + assert len(trades) == 4 trades = load_trades_from_db(db_url=default_conf['db_url'], strategy='NoneStrategy') assert len(trades) == 0 diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 5e972c694..f6e0ccd76 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -853,8 +853,8 @@ def test_api_forcebuy(botclient, mocker, fee): fee_close=fee.return_value, fee_open=fee.return_value, close_rate=0.265441, - id='22', - timeframe="5m", + id=22, + timeframe=5, strategy="DefaultStrategy" )) mocker.patch("freqtrade.rpc.RPC._rpc_forcebuy", fbuy_mock) @@ -865,7 +865,7 @@ def test_api_forcebuy(botclient, mocker, fee): assert rc.json() == { 'amount': 1, 'amount_requested': 1, - 'trade_id': '22', + 'trade_id': 22, 'close_date': None, 'close_date_hum': None, 'close_timestamp': None, @@ -907,7 +907,7 @@ def test_api_forcebuy(botclient, mocker, fee): 'sell_reason': None, 'sell_order_status': None, 'strategy': 'DefaultStrategy', - 'timeframe': '5m', + 'timeframe': 5, 'exchange': 'bittrex', } From 336dd1a29c1ac9467f4bfd5368aa83bf2f3288bc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Jan 2021 09:07:31 +0100 Subject: [PATCH 066/183] Rename api_models to api_schemas --- freqtrade/rpc/api_server/api_auth.py | 2 +- .../api_server/{api_models.py => api_schemas.py} | 0 freqtrade/rpc/api_server/api_v1.py | 16 ++++++++-------- 3 files changed, 9 insertions(+), 9 deletions(-) rename freqtrade/rpc/api_server/{api_models.py => api_schemas.py} (100%) diff --git a/freqtrade/rpc/api_server/api_auth.py b/freqtrade/rpc/api_server/api_auth.py index 8d1316906..110bb2a25 100644 --- a/freqtrade/rpc/api_server/api_auth.py +++ b/freqtrade/rpc/api_server/api_auth.py @@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException, status from fastapi.security import OAuth2PasswordBearer from fastapi.security.http import HTTPBasic, HTTPBasicCredentials -from freqtrade.rpc.api_server.api_models import AccessAndRefreshToken, AccessToken +from freqtrade.rpc.api_server.api_schemas import AccessAndRefreshToken, AccessToken from freqtrade.rpc.api_server.deps import get_api_config diff --git a/freqtrade/rpc/api_server/api_models.py b/freqtrade/rpc/api_server/api_schemas.py similarity index 100% rename from freqtrade/rpc/api_server/api_models.py rename to freqtrade/rpc/api_server/api_schemas.py diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 1a067faf4..614563d01 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -10,14 +10,14 @@ from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.data.history import get_datahandler from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC -from freqtrade.rpc.api_server.api_models import (AvailablePairs, Balances, BlacklistPayload, - BlacklistResponse, Count, Daily, DeleteTrade, - ForceBuyPayload, ForceSellPayload, Locks, Logs, - OpenTradeSchema, PairHistory, PerformanceEntry, - Ping, PlotConfig, Profit, ResultMsg, Stats, - StatusMsg, StrategyListResponse, StrategyResponse, - TradeResponse, TradeSchema, Version, - WhitelistResponse) +from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload, + BlacklistResponse, Count, Daily, DeleteTrade, + ForceBuyPayload, ForceSellPayload, Locks, Logs, + OpenTradeSchema, PairHistory, PerformanceEntry, + Ping, PlotConfig, Profit, ResultMsg, Stats, + StatusMsg, StrategyListResponse, StrategyResponse, + TradeResponse, TradeSchema, Version, + WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_rpc from freqtrade.rpc.rpc import RPCException From 3dc37dd79d1f8a9af5ffa1a9f4b5b9d46c5093f0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Jan 2021 12:54:40 +0100 Subject: [PATCH 067/183] Add types for deps --- freqtrade/rpc/api_server/deps.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/api_server/deps.py b/freqtrade/rpc/api_server/deps.py index 691759012..ce6fbf4aa 100644 --- a/freqtrade/rpc/api_server/deps.py +++ b/freqtrade/rpc/api_server/deps.py @@ -1,13 +1,17 @@ +from typing import Any, Dict + +from freqtrade.rpc.rpc import RPC + from .webserver import ApiServer -def get_rpc(): +def get_rpc() -> RPC: return ApiServer._rpc -def get_config(): +def get_config() -> Dict[str, Any]: return ApiServer._config -def get_api_config(): +def get_api_config() -> Dict[str, Any]: return ApiServer._config['api_server'] From e6176d43f3b631b6969978274baef77178f2b905 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Jan 2021 13:12:49 +0100 Subject: [PATCH 068/183] Optional RPC dependency --- freqtrade/rpc/api_server/api_v1.py | 9 ++++++--- freqtrade/rpc/api_server/deps.py | 18 ++++++++++++++---- freqtrade/rpc/api_server/webserver.py | 1 + 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 614563d01..ddca88b06 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -18,7 +18,7 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac StatusMsg, StrategyListResponse, StrategyResponse, TradeResponse, TradeSchema, Version, WhitelistResponse) -from freqtrade.rpc.api_server.deps import get_config, get_rpc +from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -100,8 +100,11 @@ def edge(rpc: RPC = Depends(get_rpc)): # TODO: Missing response model @router.get('/show_config', tags=['info']) -def show_config(rpc: RPC = Depends(get_rpc), config=Depends(get_config)): - return RPC._rpc_show_config(config, rpc._freqtrade.state) +def show_config(rpc: Optional[RPC] = Depends(get_rpc_optional), config=Depends(get_config)): + state = '' + if rpc: + state = rpc._freqtrade.state + return RPC._rpc_show_config(config, state) @router.post('/forcebuy', response_model=Union[TradeSchema, StatusMsg], tags=['trading']) diff --git a/freqtrade/rpc/api_server/deps.py b/freqtrade/rpc/api_server/deps.py index ce6fbf4aa..d2459010f 100644 --- a/freqtrade/rpc/api_server/deps.py +++ b/freqtrade/rpc/api_server/deps.py @@ -1,12 +1,22 @@ -from typing import Any, Dict +from typing import Any, Dict, Optional -from freqtrade.rpc.rpc import RPC +from freqtrade.rpc.rpc import RPC, RPCException from .webserver import ApiServer -def get_rpc() -> RPC: - return ApiServer._rpc +def get_rpc_optional() -> Optional[RPC]: + if ApiServer._has_rpc: + return ApiServer._rpc + return None + + +def get_rpc() -> Optional[RPC]: + _rpc = get_rpc_optional() + if _rpc: + return _rpc + else: + raise RPCException('Bot is not in the correct state') def get_config() -> Dict[str, Any]: diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index ac7a35a9e..a475ad494 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) class ApiServer(RPCHandler): _rpc: RPC + _has_rpc: bool = False _config: Dict[str, Any] = {} def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: From ca0bb7bbb822498cf3c219faad37d8b8f5b1c2e8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Jan 2021 15:11:40 +0100 Subject: [PATCH 069/183] Don't require RPC for strategy --- freqtrade/rpc/api_server/api_schemas.py | 10 ++++++++-- freqtrade/rpc/api_server/api_v1.py | 20 ++++++++++---------- freqtrade/rpc/api_server/webserver.py | 1 + 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 8c2a25ceb..60b2970eb 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -1,5 +1,5 @@ from datetime import date, datetime -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, TypeVar, Union from pydantic import BaseModel @@ -180,6 +180,9 @@ class TradeResponse(BaseModel): trades_count: int +ForceBuyResponse = TypeVar('ForceBuyResponse', TradeSchema, StatusMsg) + + class LockModel(BaseModel): active: bool lock_end_time: str @@ -234,11 +237,14 @@ class DeleteTrade(BaseModel): trade_id: int -class PlotConfig(BaseModel): +class PlotConfig_(BaseModel): main_plot: Dict[str, Any] subplots: Optional[Dict[str, Any]] +PlotConfig = TypeVar('PlotConfig', PlotConfig_, Dict) + + class StrategyListResponse(BaseModel): strategies: List[str] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index ddca88b06..ab68c3b68 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -1,6 +1,6 @@ from copy import deepcopy from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import List, Optional from fastapi import APIRouter, Depends from fastapi.exceptions import HTTPException @@ -12,12 +12,12 @@ from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload, BlacklistResponse, Count, Daily, DeleteTrade, - ForceBuyPayload, ForceSellPayload, Locks, Logs, - OpenTradeSchema, PairHistory, PerformanceEntry, - Ping, PlotConfig, Profit, ResultMsg, Stats, - StatusMsg, StrategyListResponse, StrategyResponse, - TradeResponse, TradeSchema, Version, - WhitelistResponse) + ForceBuyPayload, ForceBuyResponse, + ForceSellPayload, Locks, Logs, OpenTradeSchema, + PairHistory, PerformanceEntry, Ping, PlotConfig, + Profit, ResultMsg, Stats, StatusMsg, + StrategyListResponse, StrategyResponse, + TradeResponse, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -107,7 +107,7 @@ def show_config(rpc: Optional[RPC] = Depends(get_rpc_optional), config=Depends(g return RPC._rpc_show_config(config, state) -@router.post('/forcebuy', response_model=Union[TradeSchema, StatusMsg], tags=['trading']) +@router.post('/forcebuy', response_model=ForceBuyResponse, tags=['trading']) def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)): trade = rpc._rpc_forcebuy(payload.pair, payload.price) @@ -182,7 +182,7 @@ def pair_history(pair: str, timeframe: str, timerange: str, strategy: str, return RPC._rpc_analysed_history_full(config, pair, timeframe, timerange) -@router.get('/plot_config', response_model=Union[PlotConfig, Dict], tags=['candle data']) +@router.get('/plot_config', response_model=PlotConfig, tags=['candle data']) def plot_config(rpc: RPC = Depends(get_rpc)): return rpc._rpc_plot_config() @@ -199,7 +199,7 @@ def list_strategies(config=Depends(get_config)): @router.get('/strategy/{strategy}', response_model=StrategyResponse, tags=['strategy']) -def get_strategy(strategy: str, rpc: RPC = Depends(get_rpc), config=Depends(get_config)): +def get_strategy(strategy: str, config=Depends(get_config)): config = deepcopy(config) from freqtrade.resolvers.strategy_resolver import StrategyResolver diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index a475ad494..f54a96f1b 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -25,6 +25,7 @@ class ApiServer(RPCHandler): self._server = None ApiServer._rpc = rpc + ApiServer._has_rpc = True ApiServer._config = config api_config = self._config['api_server'] From cff50f9f66e68d34a05ec7a2ac732e56eb8663c3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Jan 2021 15:48:33 +0100 Subject: [PATCH 070/183] Add response-model for show_config --- freqtrade/rpc/api_server/api_schemas.py | 23 +++++++++++++++++++++++ freqtrade/rpc/api_server/api_v1.py | 5 ++--- freqtrade/rpc/rpc.py | 2 +- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 60b2970eb..16a1c263d 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -112,6 +112,29 @@ class Daily(BaseModel): stake_currency: str +class ShowConfig(BaseModel): + dry_run: str + stake_currency: str + stake_amount: float + max_open_trades: int + minimal_roi: Dict[str, Any] + stoploss: float + trailing_stop: bool + trailing_stop_positive: Optional[float] + trailing_stop_positive_offset: Optional[float] + trailing_only_offset_is_reached: Optional[bool] + timeframe: str + timeframe_ms: int + timeframe_min: int + exchange: str + strategy: str + forcebuy_enabled: bool + ask_strategy: Dict[str, Any] + bid_strategy: Dict[str, Any] + state: str + runmode: str + + class TradeSchema(BaseModel): trade_id: int pair: str diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index ab68c3b68..a2082103b 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -15,7 +15,7 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac ForceBuyPayload, ForceBuyResponse, ForceSellPayload, Locks, Logs, OpenTradeSchema, PairHistory, PerformanceEntry, Ping, PlotConfig, - Profit, ResultMsg, Stats, StatusMsg, + Profit, ResultMsg, ShowConfig, Stats, StatusMsg, StrategyListResponse, StrategyResponse, TradeResponse, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional @@ -98,8 +98,7 @@ def edge(rpc: RPC = Depends(get_rpc)): return rpc._rpc_edge() -# TODO: Missing response model -@router.get('/show_config', tags=['info']) +@router.get('/show_config', response_model=ShowConfig, tags=['info']) def show_config(rpc: Optional[RPC] = Depends(get_rpc_optional), config=Depends(get_config)): state = '' if rpc: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 70a99a186..4ad2b3485 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -111,7 +111,7 @@ class RPC: self._fiat_converter = CryptoToFiatConverter() @staticmethod - def _rpc_show_config(config, botstate: State) -> Dict[str, Any]: + def _rpc_show_config(config, botstate: Union[State, str]) -> Dict[str, Any]: """ Return a dict of config options. Explicitly does NOT return the full config to avoid leakage of sensitive From 26c34634039fcf61f51bf7e7b0e95da7334ed719 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Jan 2021 06:46:06 +0100 Subject: [PATCH 071/183] Stake-amount supports unlimited, too --- freqtrade/rpc/api_server/api_schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 16a1c263d..45f160008 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -115,7 +115,7 @@ class Daily(BaseModel): class ShowConfig(BaseModel): dry_run: str stake_currency: str - stake_amount: float + stake_amount: Union[float, str] max_open_trades: int minimal_roi: Dict[str, Any] stoploss: float From 634d6f38989288c383f98a706f3023e9a06005b4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Jan 2021 07:15:45 +0100 Subject: [PATCH 072/183] Change logging to stderr --- freqtrade/rpc/api_server/webserver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index f54a96f1b..d4c34f2f9 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -99,9 +99,14 @@ class ApiServer(RPCHandler): logger.info('Starting Local Rest Server.') verbosity = self._config['api_server'].get('verbosity', 'info') + log_config = uvicorn.config.LOGGING_CONFIG + # Change logging of access logs to stderr + log_config["handlers"]["access"]["stream"] = log_config["handlers"]["default"]["stream"] uvconfig = uvicorn.Config(self.app, port=rest_port, host=rest_ip, + use_colors=False, + log_config=log_config, access_log=True if verbosity != 'error' else False, ) try: From 5ca2cd3a1e0fe4ac51a82fc0e7439962ba697349 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Jan 2021 07:18:41 +0100 Subject: [PATCH 073/183] Change defaults to log only errors --- config.json.example | 6 +++--- config_binance.json.example | 6 +++--- config_full.json.example | 2 +- config_kraken.json.example | 6 +++--- docs/rest-api.md | 2 +- freqtrade/rpc/api_server/webserver.py | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/config.json.example b/config.json.example index af45dac74..fc59a4d5b 100644 --- a/config.json.example +++ b/config.json.example @@ -79,11 +79,11 @@ "enabled": false, "listen_ip_address": "127.0.0.1", "listen_port": 8080, - "verbosity": "info", + "verbosity": "error", "jwt_secret_key": "somethingrandom", "CORS_origins": [], - "username": "", - "password": "" + "username": "freqtrader", + "password": "SuperSecurePassword" }, "initial_state": "running", "forcebuy_enable": false, diff --git a/config_binance.json.example b/config_binance.json.example index f3f8eb659..954634def 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -84,11 +84,11 @@ "enabled": false, "listen_ip_address": "127.0.0.1", "listen_port": 8080, - "verbosity": "info", + "verbosity": "error", "jwt_secret_key": "somethingrandom", "CORS_origins": [], - "username": "", - "password": "" + "username": "freqtrader", + "password": "SuperSecurePassword" }, "initial_state": "running", "forcebuy_enable": false, diff --git a/config_full.json.example b/config_full.json.example index ef9f45363..7cdd6af67 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -164,7 +164,7 @@ "enabled": false, "listen_ip_address": "127.0.0.1", "listen_port": 8080, - "verbosity": "info", + "verbosity": "error", "enable_openapi": false, "jwt_secret_key": "somethingrandom", "CORS_origins": [], diff --git a/config_kraken.json.example b/config_kraken.json.example index 5f3b57854..4b33eb592 100644 --- a/config_kraken.json.example +++ b/config_kraken.json.example @@ -89,11 +89,11 @@ "enabled": false, "listen_ip_address": "127.0.0.1", "listen_port": 8080, - "verbosity": "info", + "verbosity": "error", "jwt_secret_key": "somethingrandom", "CORS_origins": [], - "username": "", - "password": "" + "username": "freqtrader", + "password": "SuperSecurePassword" }, "initial_state": "running", "forcebuy_enable": false, diff --git a/docs/rest-api.md b/docs/rest-api.md index 279373c50..a013bf358 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -11,7 +11,7 @@ Sample configuration: "enabled": true, "listen_ip_address": "127.0.0.1", "listen_port": 8080, - "verbosity": "info", + "verbosity": "error", "enable_openapi": false, "jwt_secret_key": "somethingrandom", "CORS_origins": [], diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index d4c34f2f9..97dfa444d 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -31,7 +31,7 @@ class ApiServer(RPCHandler): self.app = FastAPI(title="Freqtrade API", openapi_url='openapi.json' if api_config.get( - 'enable_openapi') else None, + 'enable_openapi', False) else None, redoc_url=None, ) self.configure_app(self.app, self._config) @@ -98,7 +98,7 @@ class ApiServer(RPCHandler): "Others may be able to log into your bot.") logger.info('Starting Local Rest Server.') - verbosity = self._config['api_server'].get('verbosity', 'info') + verbosity = self._config['api_server'].get('verbosity', 'error') log_config = uvicorn.config.LOGGING_CONFIG # Change logging of access logs to stderr log_config["handlers"]["access"]["stream"] = log_config["handlers"]["default"]["stream"] From 66391b80ae6c8b87c96e819e894e32d4c83b167e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Jan 2021 05:38:57 +0000 Subject: [PATCH 074/183] Bump isort from 5.6.4 to 5.7.0 Bumps [isort](https://github.com/pycqa/isort) from 5.6.4 to 5.7.0. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/develop/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.6.4...5.7.0) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index a2da87430..e918a9b90 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,7 +13,7 @@ pytest-asyncio==0.14.0 pytest-cov==2.10.1 pytest-mock==3.4.0 pytest-random-order==1.0.4 -isort==5.6.4 +isort==5.7.0 # Convert jupyter notebooks to markdown documents nbconvert==6.0.7 From 7d06e61461b65be63d99e17a62828fd1e1c63947 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Jan 2021 05:38:57 +0000 Subject: [PATCH 075/183] Bump scipy from 1.5.4 to 1.6.0 Bumps [scipy](https://github.com/scipy/scipy) from 1.5.4 to 1.6.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.5.4...v1.6.0) Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index a2446ddb8..fbb963cf9 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,7 +2,7 @@ -r requirements.txt # Required for hyperopt -scipy==1.5.4 +scipy==1.6.0 scikit-learn==0.24.0 scikit-optimize==0.8.1 filelock==3.0.12 From 9e435fba0bd22e2515e9f01b6aa6c2dc9e9fc721 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Jan 2021 05:39:14 +0000 Subject: [PATCH 076/183] Bump ccxt from 1.39.79 to 1.40.14 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.39.79 to 1.40.14. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst) - [Commits](https://github.com/ccxt/ccxt/compare/1.39.79...1.40.14) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bab13ed03..383c9686e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.19.4 pandas==1.2.0 -ccxt==1.39.79 +ccxt==1.40.14 aiohttp==3.7.3 SQLAlchemy==1.3.22 python-telegram-bot==13.1 From d1804dee6b2675e1353919eef71f965a91093e05 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 4 Jan 2021 09:40:17 +0100 Subject: [PATCH 077/183] Add note about python-dev dependency --- docs/installation.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index be98c45a8..73e791c56 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -34,7 +34,8 @@ The easiest way to install and run Freqtrade is to clone the bot Github reposito When cloning the repository the default working branch has the name `develop`. This branch contains all last features (can be considered as relatively stable, thanks to automated tests). The `stable` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable). !!! Note - Python3.7 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository. + Python3.7 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository. + Also, python headers (`python-dev` / `python-devel`) must be available for the installation to complete successfully. This can be achieved with the following commands: From 07bc0c3fce4b3ed6557e93b251025ce9e2119387 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 4 Jan 2021 13:47:16 +0100 Subject: [PATCH 078/183] Improve merge_informative_pairs to properly merge correct timeframes explanation in #4073, closes #4073 --- freqtrade/strategy/strategy_helper.py | 13 +++++++++++-- tests/strategy/test_strategy_helpers.py | 15 ++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index ea0e234ec..d7b1327d9 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -24,15 +24,24 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, :param timeframe: Timeframe of the original pair sample. :param timeframe_inf: Timeframe of the informative pair sample. :param ffill: Forwardfill missing values - optional but usually required + :return: Merged dataframe + :raise: ValueError if the secondary timeframe is shorter than the dataframe timeframe """ minutes_inf = timeframe_to_minutes(timeframe_inf) minutes = timeframe_to_minutes(timeframe) - if minutes >= minutes_inf: + if minutes == minutes_inf: # No need to forwardshift if the timeframes are identical informative['date_merge'] = informative["date"] + elif minutes < minutes_inf: + # Subtract "small" timeframe so merging is not delayed by 1 small candle + # Detailed explanation in https://github.com/freqtrade/freqtrade/issues/4073 + informative['date_merge'] = ( + informative["date"] + pd.to_timedelta(minutes_inf, 'm') - pd.to_timedelta(minutes, 'm') + ) else: - informative['date_merge'] = informative["date"] + pd.to_timedelta(minutes_inf, 'm') + raise ValueError("Tried to merge a faster timeframe to a slower timeframe." + "This would create new rows, and can throw off your regular indicators.") # Rename columns to be unique informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns] diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 1d3e80d24..252288e2e 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd +import pytest from freqtrade.strategy import merge_informative_pair, timeframe_to_minutes @@ -47,17 +48,17 @@ def test_merge_informative_pair(): assert 'volume_1h' in result.columns assert result['volume'].equals(data['volume']) - # First 4 rows are empty + # First 3 rows are empty assert result.iloc[0]['date_1h'] is pd.NaT assert result.iloc[1]['date_1h'] is pd.NaT assert result.iloc[2]['date_1h'] is pd.NaT - assert result.iloc[3]['date_1h'] is pd.NaT # Next 4 rows contain the starting date (0:00) + assert result.iloc[3]['date_1h'] == result.iloc[0]['date'] assert result.iloc[4]['date_1h'] == result.iloc[0]['date'] assert result.iloc[5]['date_1h'] == result.iloc[0]['date'] assert result.iloc[6]['date_1h'] == result.iloc[0]['date'] - assert result.iloc[7]['date_1h'] == result.iloc[0]['date'] # Next 4 rows contain the next Hourly date original date row 4 + assert result.iloc[7]['date_1h'] == result.iloc[4]['date'] assert result.iloc[8]['date_1h'] == result.iloc[4]['date'] @@ -86,3 +87,11 @@ def test_merge_informative_pair_same(): # Dates match 1:1 assert result['date_15m'].equals(result['date']) + + +def test_merge_informative_pair_lower(): + data = generate_test_data('1h', 40) + informative = generate_test_data('15m', 40) + + with pytest.raises(ValueError, match=r"Tried to merge a faster timeframe .*"): + merge_informative_pair(data, informative, '1h', '15m', ffill=True) From 0704cfb05b611a2f88bf5b44d02df273d1d8e628 Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Mon, 4 Jan 2021 14:14:52 +0100 Subject: [PATCH 079/183] Added an example with a positive offset for a custom stoploss Signed-off-by: hoeckxer --- docs/strategy-advanced.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index b1fcb50fc..9d5930518 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -119,6 +119,41 @@ class AwesomeStrategy(IStrategy): return -0.15 ``` +#### Trailing stoploss with positive offset + +Use the initial stoploss until the profit is above 4%, then use a trailing stoploss of 50% of the current profit with a minimum of 2.5% and a maximum of 5%. + +Please note that the stoploss can only increase, values lower than the current stoploss are ignored. + +``` python +from datetime import datetime, timedelta +from freqtrade.persistence import Trade + +class AwesomeStrategy(IStrategy): + + # ... populate_* methods + + use_custom_stoploss = True + + def custom_stoploss(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> float: + + if current_profit < 0.04: + return -1 # return a value bigger than the inital stoploss to keep using the inital stoploss + + # After reaching the desired offset, allow the stoploss to trail by half the profit + stoploss = current_profit / 2 + + if abs(stoploss) < 0.025: + # Maintain a minimum of 2.5% trailing stoploss + stoploss = 0.025 + if abs(stoploss) > 0.05: + # Maximize the stoploss at 5% + stoploss = 0.05 + + return stoploss +``` + #### Absolute stoploss The below example sets absolute profit levels based on the current profit. From 1cf6e2c957de86054d5611ddf78230754e3fb112 Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Mon, 4 Jan 2021 14:37:22 +0100 Subject: [PATCH 080/183] Changed documentation based on review comments Signed-off-by: hoeckxer --- docs/strategy-advanced.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 9d5930518..2431274d7 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -142,16 +142,10 @@ class AwesomeStrategy(IStrategy): return -1 # return a value bigger than the inital stoploss to keep using the inital stoploss # After reaching the desired offset, allow the stoploss to trail by half the profit - stoploss = current_profit / 2 + desired_stoploss = current_profit / 2 - if abs(stoploss) < 0.025: - # Maintain a minimum of 2.5% trailing stoploss - stoploss = 0.025 - if abs(stoploss) > 0.05: - # Maximize the stoploss at 5% - stoploss = 0.05 - - return stoploss + # Use a minimum of 2.5% and a maximum of 5% + return max(min(desired_stoploss, 0.05), 0.025) ``` #### Absolute stoploss From 614a99659762c2b5594fcca1ddfc1c1458a732ae Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Mon, 4 Jan 2021 20:49:24 +0100 Subject: [PATCH 081/183] First commit about ignoring expired candle Signed-off-by: hoeckxer --- docs/configuration.md | 17 +++++++++++++++++ freqtrade/strategy/interface.py | 21 ++++++++++++++++++++- tests/strategy/test_interface.py | 17 +++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index b70a85c04..db078cba8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -121,6 +121,8 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `user_data_dir` | Directory containing user data.
*Defaults to `./user_data/`*.
**Datatype:** String | `dataformat_ohlcv` | Data format to use to store historical candle (OHLCV) data.
*Defaults to `json`*.
**Datatype:** String | `dataformat_trades` | Data format to use to store historical trades data.
*Defaults to `jsongz`*.
**Datatype:** String +| `ignore_buying_expired_candle` | Enables usage of skipping buys on candles that are older than a specified period.
*Defaults to `False`*
**Datatype:** Boolean +| `ignore_buying_expired_candle_after` | Specifies the number of seconds until a buy signal is no longer used when setting `ignore_buying_expired_candle`.
**Datatype:** Integer ### Parameters in the strategy @@ -144,6 +146,8 @@ Values set in the configuration file always overwrite values set in the strategy * `use_sell_signal` (ask_strategy) * `sell_profit_only` (ask_strategy) * `ignore_roi_if_buy_signal` (ask_strategy) +* `ignore_buying_expired_candle` +* `ignore_buying_expired_candle_after` ### Configuring amount per trade @@ -671,6 +675,19 @@ export HTTPS_PROXY="http://addr:port" freqtrade ``` +## Ignoring expired candles + +When working with larger timeframes (for example 1h or more) and using a low `max_trades` value, the last candle can be processed as soon as a trade slot becomes available. When processing the last candle, this can lead to a situation where it may not be desirable to use the buy signal on that candle. For example, when using a condition in your strategy where you use a cross-over, that point may have passed too long ago for you to start a trade on it. + +In these situations, you can enable the functionality to ignore candles that are beyond a specified period by setting `ignore_buying_expired_candle` to `True`. After this, you can set `ignore_buying_expired_candle_after` to the number of seconds after which the candle becomes expired. + +For example, if your strategy is using a 1h timeframe, and you only want to buy within the first 5 minutes when a new candle comes in, you can add the following configuration to your strategy: + +``` json +ignore_buying_expired_candle = True +ignore_buying_expired_candle_after = 300 # 5 minutes +``` + ## Embedding Strategies Freqtrade provides you with with an easy way to embed the strategy into your configuration file. diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 1e4fc8b12..62ef4e91b 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -5,7 +5,7 @@ This module defines the interface to apply for strategies import logging import warnings from abc import ABC, abstractmethod -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from enum import Enum from typing import Dict, List, NamedTuple, Optional, Tuple @@ -113,6 +113,11 @@ class IStrategy(ABC): # run "populate_indicators" only for new candle process_only_new_candles: bool = False + # Don't analyze too old candles + ignore_buying_expired_candle: bool = False + # Number of seconds after which the candle will no longer result in a buy + ignore_buying_expired_candle_after: int = 0 + # Disable checking the dataframe (converts the error into a warning message) disable_dataframe_checks: bool = False @@ -476,8 +481,21 @@ class IStrategy(ABC): (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 logger.debug('trigger: %s (pair=%s) buy=%s sell=%s', latest['date'], pair, str(buy), str(sell)) + if self.ignore_expired_candle(dataframe=dataframe, buy=buy): + return False, sell return buy, sell + def ignore_expired_candle(self, dataframe: DataFrame, buy: bool): + if self.ignore_buying_expired_candle and buy: + current_time = datetime.now(timezone.utc) - timedelta(seconds=self.ignore_buying_expired_candle_after) + candle_time = dataframe['date'].tail(1).iat[0] + time_delta = current_time - candle_time + if time_delta.total_seconds() > self.ignore_buying_expired_candle_after: + logger.debug('ignoring buy signals because candle exceeded ignore_buying_expired_candle_after of %s seconds', self.ignore_buying_expired_candle_after) + return True + else: + return False + def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, sell: bool, low: float = None, high: float = None, force_stoploss: float = 0) -> SellCheckTuple: @@ -672,6 +690,7 @@ class IStrategy(ABC): :return: DataFrame with buy column """ logger.debug(f"Populating buy signals for pair {metadata.get('pair')}.") + if self._buy_fun_len == 2: warnings.warn("deprecated - check out the Sample strategy to see " "the current function headers!", DeprecationWarning) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 7eed43302..330689039 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -106,6 +106,23 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history): assert log_has('Outdated history for pair xyz. Last tick is 16 minutes old', caplog) +def test_ignore_expired_candle(default_conf, ohlcv_history): + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) + strategy.ignore_buying_expired_candle = True + strategy.ignore_buying_expired_candle_after = 60 + + ohlcv_history.loc[-1, 'date'] = arrow.utcnow().shift(minutes=-3) + # Take a copy to correctly modify the call + mocked_history = ohlcv_history.copy() + mocked_history['sell'] = 0 + mocked_history['buy'] = 0 + mocked_history.loc[1, 'buy'] = 1 + mocked_history.loc[1, 'sell'] = 1 + + assert strategy.ignore_expired_candle(mocked_history, True) == True + + def test_assert_df_raise(mocker, caplog, ohlcv_history): ohlcv_history.loc[1, 'date'] = arrow.utcnow().shift(minutes=-16) # Take a copy to correctly modify the call From 844df96ec77b7e9f28969627e26bc53253189f76 Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Tue, 5 Jan 2021 07:06:53 +0100 Subject: [PATCH 082/183] Making changes so the build checks are satisified (imports & flake8) Signed-off-by: hoeckxer --- freqtrade/strategy/interface.py | 10 +++++++--- tests/strategy/test_interface.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 62ef4e91b..5f7ef8590 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -5,7 +5,7 @@ This module defines the interface to apply for strategies import logging import warnings from abc import ABC, abstractmethod -from datetime import datetime, timezone, timedelta +from datetime import datetime, timedelta, timezone from enum import Enum from typing import Dict, List, NamedTuple, Optional, Tuple @@ -487,11 +487,15 @@ class IStrategy(ABC): def ignore_expired_candle(self, dataframe: DataFrame, buy: bool): if self.ignore_buying_expired_candle and buy: - current_time = datetime.now(timezone.utc) - timedelta(seconds=self.ignore_buying_expired_candle_after) + current_time = datetime.now(timezone.utc) - timedelta( + seconds=self.ignore_buying_expired_candle_after) candle_time = dataframe['date'].tail(1).iat[0] time_delta = current_time - candle_time if time_delta.total_seconds() > self.ignore_buying_expired_candle_after: - logger.debug('ignoring buy signals because candle exceeded ignore_buying_expired_candle_after of %s seconds', self.ignore_buying_expired_candle_after) + logger.debug( + '''ignoring buy signals because candle exceeded + ignore_buying_expired_candle_after of %s seconds''', + self.ignore_buying_expired_candle_after) return True else: return False diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 330689039..f389be45b 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -120,7 +120,7 @@ def test_ignore_expired_candle(default_conf, ohlcv_history): mocked_history.loc[1, 'buy'] = 1 mocked_history.loc[1, 'sell'] = 1 - assert strategy.ignore_expired_candle(mocked_history, True) == True + assert strategy.ignore_expired_candle(mocked_history, True) is True def test_assert_df_raise(mocker, caplog, ohlcv_history): From 9a93a0876ab7080cfa4eb424fff8a1f7eeb92d29 Mon Sep 17 00:00:00 2001 From: Erwin Hoeckx Date: Tue, 5 Jan 2021 07:32:07 +0100 Subject: [PATCH 083/183] Update interface.py Adjusted comment --- freqtrade/strategy/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 5f7ef8590..4d6e327f3 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -113,7 +113,7 @@ class IStrategy(ABC): # run "populate_indicators" only for new candle process_only_new_candles: bool = False - # Don't analyze too old candles + # Don't buy on expired candles ignore_buying_expired_candle: bool = False # Number of seconds after which the candle will no longer result in a buy ignore_buying_expired_candle_after: int = 0 From 67306d943a0e7b0a86d799cc6022ab117358369b Mon Sep 17 00:00:00 2001 From: Erwin Hoeckx Date: Tue, 5 Jan 2021 07:33:34 +0100 Subject: [PATCH 084/183] Update interface.py Simplified return value, thereby including the situation where the time simply hasn't expired yet --- freqtrade/strategy/interface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4d6e327f3..41fa26ac4 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -497,8 +497,8 @@ class IStrategy(ABC): ignore_buying_expired_candle_after of %s seconds''', self.ignore_buying_expired_candle_after) return True - else: - return False + + return False def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, sell: bool, low: float = None, high: float = None, From c9ed2137bb3b324a40d70110e0d5b959744fac10 Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Tue, 5 Jan 2021 09:07:46 +0100 Subject: [PATCH 085/183] Simplified return statements Signed-off-by: hoeckxer --- freqtrade/strategy/interface.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 41fa26ac4..a1da3681d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -491,14 +491,9 @@ class IStrategy(ABC): seconds=self.ignore_buying_expired_candle_after) candle_time = dataframe['date'].tail(1).iat[0] time_delta = current_time - candle_time - if time_delta.total_seconds() > self.ignore_buying_expired_candle_after: - logger.debug( - '''ignoring buy signals because candle exceeded - ignore_buying_expired_candle_after of %s seconds''', - self.ignore_buying_expired_candle_after) - return True - - return False + return time_delta.total_seconds() > self.ignore_buying_expired_candle_after + else: + return False def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, sell: bool, low: float = None, high: float = None, From eaaaddac86cf1f480fc34c6a0d8d33f3bffbceea Mon Sep 17 00:00:00 2001 From: Erwin Hoeckx Date: Tue, 5 Jan 2021 11:10:00 +0100 Subject: [PATCH 086/183] Update docs/configuration.md Co-authored-by: Matthias --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index db078cba8..c1c2e65ec 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -677,7 +677,7 @@ freqtrade ## Ignoring expired candles -When working with larger timeframes (for example 1h or more) and using a low `max_trades` value, the last candle can be processed as soon as a trade slot becomes available. When processing the last candle, this can lead to a situation where it may not be desirable to use the buy signal on that candle. For example, when using a condition in your strategy where you use a cross-over, that point may have passed too long ago for you to start a trade on it. +When working with larger timeframes (for example 1h or more) and using a low `max_open_trades` value, the last candle can be processed as soon as a trade slot becomes available. When processing the last candle, this can lead to a situation where it may not be desirable to use the buy signal on that candle. For example, when using a condition in your strategy where you use a cross-over, that point may have passed too long ago for you to start a trade on it. In these situations, you can enable the functionality to ignore candles that are beyond a specified period by setting `ignore_buying_expired_candle` to `True`. After this, you can set `ignore_buying_expired_candle_after` to the number of seconds after which the candle becomes expired. From e3f3f3629828242167aac1d7cd65939fa60fbfd6 Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Tue, 5 Jan 2021 14:49:35 +0100 Subject: [PATCH 087/183] Changes based on review comments --- freqtrade/resolvers/strategy_resolver.py | 2 ++ freqtrade/strategy/interface.py | 10 +++++----- tests/strategy/test_interface.py | 9 +++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 73af00fee..5872d95a6 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -79,6 +79,8 @@ class StrategyResolver(IResolver): ("sell_profit_only", False, 'ask_strategy'), ("ignore_roi_if_buy_signal", False, 'ask_strategy'), ("disable_dataframe_checks", False, None), + ("ignore_buying_expired_candle", None, 'ask_strategy'), + ("ignore_buying_expired_candle_after", 0, 'ask_strategy') ] for attribute, default, subkey in attributes: if subkey: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index a1da3681d..8976b2fd5 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -15,7 +15,7 @@ from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider from freqtrade.exceptions import OperationalException, StrategyError -from freqtrade.exchange import timeframe_to_minutes +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange.exchange import timeframe_to_next_date from freqtrade.persistence import PairLocks, Trade from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper @@ -481,16 +481,16 @@ class IStrategy(ABC): (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 logger.debug('trigger: %s (pair=%s) buy=%s sell=%s', latest['date'], pair, str(buy), str(sell)) - if self.ignore_expired_candle(dataframe=dataframe, buy=buy): + timeframe_seconds = timeframe_to_seconds(timeframe) + if self.ignore_expired_candle(latest_date=latest_date, timeframe_seconds=timeframe_seconds, buy=buy): return False, sell return buy, sell - def ignore_expired_candle(self, dataframe: DataFrame, buy: bool): + def ignore_expired_candle(self, latest_date: datetime, timeframe_seconds: int, buy: bool): if self.ignore_buying_expired_candle and buy: current_time = datetime.now(timezone.utc) - timedelta( seconds=self.ignore_buying_expired_candle_after) - candle_time = dataframe['date'].tail(1).iat[0] - time_delta = current_time - candle_time + time_delta = current_time - latest_date + timedelta(seconds=timeframe_seconds) return time_delta.total_seconds() > self.ignore_buying_expired_candle_after else: return False diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index f389be45b..af086b0da 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -112,15 +112,12 @@ def test_ignore_expired_candle(default_conf, ohlcv_history): strategy.ignore_buying_expired_candle = True strategy.ignore_buying_expired_candle_after = 60 - ohlcv_history.loc[-1, 'date'] = arrow.utcnow().shift(minutes=-3) + ohlcv_history.loc[-1, 'date'] = arrow.utcnow() # Take a copy to correctly modify the call mocked_history = ohlcv_history.copy() - mocked_history['sell'] = 0 - mocked_history['buy'] = 0 - mocked_history.loc[1, 'buy'] = 1 - mocked_history.loc[1, 'sell'] = 1 + latest_date = mocked_history['date'].max() - assert strategy.ignore_expired_candle(mocked_history, True) is True + assert strategy.ignore_expired_candle(latest_date=latest_date, timeframe_seconds=300, buy=True) is True def test_assert_df_raise(mocker, caplog, ohlcv_history): From 573de1cf08a323accee427bd168a8bbd0fec7430 Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Tue, 5 Jan 2021 15:30:29 +0100 Subject: [PATCH 088/183] Fixed flake8 warnings --- freqtrade/strategy/interface.py | 5 +++-- tests/strategy/test_interface.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 8976b2fd5..2b4a8dd03 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -21,7 +21,6 @@ from freqtrade.persistence import PairLocks, Trade from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets - logger = logging.getLogger(__name__) @@ -482,7 +481,9 @@ class IStrategy(ABC): logger.debug('trigger: %s (pair=%s) buy=%s sell=%s', latest['date'], pair, str(buy), str(sell)) timeframe_seconds = timeframe_to_seconds(timeframe) - if self.ignore_expired_candle(latest_date=latest_date, timeframe_seconds=timeframe_seconds, buy=buy): + if self.ignore_expired_candle(latest_date=latest_date, + timeframe_seconds=timeframe_seconds, + buy=buy): return False, sell return buy, sell diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index af086b0da..d7d113a4e 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -117,7 +117,9 @@ def test_ignore_expired_candle(default_conf, ohlcv_history): mocked_history = ohlcv_history.copy() latest_date = mocked_history['date'].max() - assert strategy.ignore_expired_candle(latest_date=latest_date, timeframe_seconds=300, buy=True) is True + assert strategy.ignore_expired_candle(latest_date=latest_date, + timeframe_seconds=300, + buy=True) is True def test_assert_df_raise(mocker, caplog, ohlcv_history): From 65d91a3a58c7807f2e9b18a85d780ba7ea4dc6a4 Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Tue, 5 Jan 2021 15:36:34 +0100 Subject: [PATCH 089/183] isort fix --- freqtrade/strategy/interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 2b4a8dd03..a18c6c915 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -21,6 +21,7 @@ from freqtrade.persistence import PairLocks, Trade from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets + logger = logging.getLogger(__name__) From 5c34140a191a96a82e8ca617bf10d4b16f909b90 Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Tue, 5 Jan 2021 20:59:31 +0100 Subject: [PATCH 090/183] Adjusted documentation to reflect sub-key configuration --- docs/configuration.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c1c2e65ec..6ae37e7b3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -74,6 +74,8 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `ask_strategy.use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
**Datatype:** Boolean | `ask_strategy.sell_profit_only` | Wait until the bot makes a positive profit before taking a sell decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `ask_strategy.ignore_roi_if_buy_signal` | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `ask_strategy.ignore_buying_expired_candle` | Enables usage of skipping buys on candles that are older than a specified period.
*Defaults to `False`*
**Datatype:** Boolean +| `ask_strategy.ignore_buying_expired_candle_after` | Specifies the number of seconds until a buy signal is no longer used when setting `ignore_buying_expired_candle`.
**Datatype:** Integer | `order_types` | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict | `order_time_in_force` | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict | `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
**Datatype:** String @@ -121,8 +123,6 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `user_data_dir` | Directory containing user data.
*Defaults to `./user_data/`*.
**Datatype:** String | `dataformat_ohlcv` | Data format to use to store historical candle (OHLCV) data.
*Defaults to `json`*.
**Datatype:** String | `dataformat_trades` | Data format to use to store historical trades data.
*Defaults to `jsongz`*.
**Datatype:** String -| `ignore_buying_expired_candle` | Enables usage of skipping buys on candles that are older than a specified period.
*Defaults to `False`*
**Datatype:** Boolean -| `ignore_buying_expired_candle_after` | Specifies the number of seconds until a buy signal is no longer used when setting `ignore_buying_expired_candle`.
**Datatype:** Integer ### Parameters in the strategy @@ -146,8 +146,8 @@ Values set in the configuration file always overwrite values set in the strategy * `use_sell_signal` (ask_strategy) * `sell_profit_only` (ask_strategy) * `ignore_roi_if_buy_signal` (ask_strategy) -* `ignore_buying_expired_candle` -* `ignore_buying_expired_candle_after` +* `ignore_buying_expired_candle` (ask_strategy) +* `ignore_buying_expired_candle_after` (ask_strategy) ### Configuring amount per trade @@ -679,13 +679,17 @@ freqtrade When working with larger timeframes (for example 1h or more) and using a low `max_open_trades` value, the last candle can be processed as soon as a trade slot becomes available. When processing the last candle, this can lead to a situation where it may not be desirable to use the buy signal on that candle. For example, when using a condition in your strategy where you use a cross-over, that point may have passed too long ago for you to start a trade on it. -In these situations, you can enable the functionality to ignore candles that are beyond a specified period by setting `ignore_buying_expired_candle` to `True`. After this, you can set `ignore_buying_expired_candle_after` to the number of seconds after which the candle becomes expired. +In these situations, you can enable the functionality to ignore candles that are beyond a specified period by setting `ask_strategy.ignore_buying_expired_candle` to `true`. After this, you can set `ask_strategy.ignore_buying_expired_candle_after` to the number of seconds after which the candle becomes expired. For example, if your strategy is using a 1h timeframe, and you only want to buy within the first 5 minutes when a new candle comes in, you can add the following configuration to your strategy: -``` json -ignore_buying_expired_candle = True -ignore_buying_expired_candle_after = 300 # 5 minutes +``` jsonc + "ask_strategy":{ + "ignore_buying_expired_candle" = true + "ignore_buying_expired_candle_after" = 300 # 5 minutes + "price_side": "bid", + // ... + }, ``` ## Embedding Strategies From 95732e8991094668ab2cc43224503177709905a9 Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Tue, 5 Jan 2021 21:03:23 +0100 Subject: [PATCH 091/183] Clarification in documentation Signed-off-by: hoeckxer --- docs/includes/protections.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 87db17fd8..7d2107bee 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -62,9 +62,9 @@ The below example stops trading for all pairs for 4 candles after the last trade #### MaxDrawdown -`MaxDrawdown` uses all trades within `lookback_period` (in minutes) to determine the maximum drawdown. If the drawdown is below `max_allowed_drawdown`, trading will stop for `stop_duration` (in minutes) after the last trade - assuming that the bot needs some time to let markets recover. +`MaxDrawdown` uses all trades within `lookback_period` (in minutes or candles) to determine the maximum drawdown. If the drawdown is below `max_allowed_drawdown`, trading will stop for `stop_duration` (in minutes or candles) after the last trade - assuming that the bot needs some time to let markets recover. -The below sample stops trading for 12 candles if max-drawdown is > 20% considering all trades within the last 48 candles. +The below sample stops trading for 12 candles if max-drawdown is > 20% considering all trades within the last 48 candles. If desired, `lookback_period_minutes` and/or `stop_duration_minutes` can be used. ```json "protections": [ From f7b055a58c9573778af3f517cc57b36873cf2e1a Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Wed, 6 Jan 2021 09:26:03 +0100 Subject: [PATCH 092/183] Attempt to improve wording Signed-off-by: hoeckxer --- docs/includes/protections.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 7d2107bee..560100e4d 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -62,9 +62,9 @@ The below example stops trading for all pairs for 4 candles after the last trade #### MaxDrawdown -`MaxDrawdown` uses all trades within `lookback_period` (in minutes or candles) to determine the maximum drawdown. If the drawdown is below `max_allowed_drawdown`, trading will stop for `stop_duration` (in minutes or candles) after the last trade - assuming that the bot needs some time to let markets recover. +`MaxDrawdown` uses all trades within `lookback_period` in minutes (or in candles when using `lookback_period_candles`) to determine the maximum drawdown. If the drawdown is below `max_allowed_drawdown`, trading will stop for `stop_duration` in minutes (or in candles when using `stop_duration_candles`) after the last trade - assuming that the bot needs some time to let markets recover. -The below sample stops trading for 12 candles if max-drawdown is > 20% considering all trades within the last 48 candles. If desired, `lookback_period_minutes` and/or `stop_duration_minutes` can be used. +The below sample stops trading for 12 candles if max-drawdown is > 20% considering all trades within the last 48 candles. If desired, `lookback_period` and/or `stop_duration` can be used. ```json "protections": [ From a90609315307583738fee87dd565cec799edfbfd Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Jan 2021 09:45:21 +0100 Subject: [PATCH 093/183] FIx doc wording for all guards --- docs/includes/protections.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 560100e4d..b98a0d662 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -39,7 +39,7 @@ All protection end times are rounded up to the next candle to avoid sudden, unex #### Stoploss Guard -`StoplossGuard` selects all trades within `lookback_period`, and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration`. +`StoplossGuard` selects all trades within `lookback_period` in minutes (or in candles when using `lookback_period_candles`), and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration` in minutes (or in candles when using `stop_duration_candles`). This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time. The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. @@ -81,8 +81,8 @@ The below sample stops trading for 12 candles if max-drawdown is > 20% consideri #### Low Profit Pairs -`LowProfitPairs` uses all trades for a pair within `lookback_period` (in minutes) to determine the overall profit ratio. -If that ratio is below `required_profit`, that pair will be locked for `stop_duration` (in minutes). +`LowProfitPairs` uses all trades for a pair within `lookback_period` in minutes (or in candles when using `lookback_period_candles`) to determine the overall profit ratio. +If that ratio is below `required_profit`, that pair will be locked for `stop_duration` in minutes (or in candles when using `stop_duration_candles`). The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 candles. @@ -100,7 +100,7 @@ The below example will stop trading a pair for 60 minutes if the pair does not h #### Cooldown Period -`CooldownPeriod` locks a pair for `stop_duration` (in minutes) after selling, avoiding a re-entry for this pair for `stop_duration` minutes. +`CooldownPeriod` locks a pair for `stop_duration` in minutes (or in candles when using `stop_duration_candles`) after selling, avoiding a re-entry for this pair for `stop_duration` minutes. The below example will stop trading a pair for 2 candles after closing a trade, allowing this pair to "cool down". From 91f86678817975d8c35d32bc502103a45e601a0b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Jan 2021 09:57:36 +0100 Subject: [PATCH 094/183] DOn't update open orders in dry-run mode --- freqtrade/freqtradebot.py | 5 +++++ tests/test_freqtradebot.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d60b111f2..6dc8eacf9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -246,6 +246,10 @@ class FreqtradeBot(LoggingMixin): Updates open orders based on order list kept in the database. Mainly updates the state of orders - but may also close trades """ + if self.config['dry_run']: + # Updating open orders in dry-run does not make sense and will fail. + return + orders = Order.get_open_orders() logger.info(f"Updating {len(orders)} open orders.") for order in orders: @@ -256,6 +260,7 @@ class FreqtradeBot(LoggingMixin): self.update_trade_state(order.trade, order.order_id, fo) except ExchangeError as e: + logger.warning(f"Error updating Order {order.order_id} due to {e}") def update_closed_trades_without_assigned_fees(self): diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 12be5ae8b..5c5666788 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -4313,6 +4313,11 @@ def test_update_open_orders(mocker, default_conf, fee, caplog): create_mock_trades(fee) freqtrade.update_open_orders() + assert not log_has_re(r"Error updating Order .*", caplog) + + freqtrade.config['dry_run'] = False + freqtrade.update_open_orders() + assert log_has_re(r"Error updating Order .*", caplog) caplog.clear() From a9ca72c1b8e489117d9e737592da7240bc3ef1b0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Jan 2021 11:04:14 +0100 Subject: [PATCH 095/183] Fix typo in documentation --- docs/docker_quickstart.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 48ee34954..e25e1b050 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -123,7 +123,7 @@ Advanced users may edit the docker-compose file further to include all possible All possible freqtrade arguments will be available by running `docker-compose run --rm freqtrade `. !!! Note "`docker-compose run --rm`" - Including `--rm` will clean up the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command). + Including `--rm` will remove the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command). #### Example: Download data with docker-compose @@ -172,19 +172,19 @@ docker-compose run --rm freqtrade plot-dataframe --strategy AwesomeStrategy -p B The output will be stored in the `user_data/plot` directory, and can be opened with any modern browser. -## Data analayis using docker compose +## Data analysis using docker compose Freqtrade provides a docker-compose file which starts up a jupyter lab server. You can run this server using the following command: ``` bash -docker-compose --rm -f docker/docker-compose-jupyter.yml up +docker-compose -f docker/docker-compose-jupyter.yml up ``` -This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`. +This will create a docker-container running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`. Please use the link that's printed in the console after startup for simplified login. -Since part of this image is built on your machine, it is recommended to rebuild the image from time to time to keep freqtrade (and dependencies) uptodate. +Since part of this image is built on your machine, it is recommended to rebuild the image from time to time to keep freqtrade (and dependencies) up-to-date. ``` bash docker-compose -f docker/docker-compose-jupyter.yml build --no-cache From e69dac27043368af4e0cd0b81c21ca494adf2bd1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Jan 2021 15:38:46 +0100 Subject: [PATCH 096/183] Fix bug in RPC history mode when no data is found --- freqtrade/rpc/rpc.py | 2 ++ tests/rpc/test_rpc_apiserver.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 4ad2b3485..19c90fff0 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -788,6 +788,8 @@ class RPC: timerange=timerange_parsed, data_format=config.get('dataformat_ohlcv', 'json'), ) + if pair not in _data: + raise RPCException(f"No data for {pair}, {timeframe} in {timerange} found.") from freqtrade.resolvers.strategy_resolver import StrategyResolver strategy = StrategyResolver.load_strategy(config) df_analyzed = strategy.analyze_ticker(_data[pair], {'pair': pair}) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index f6e0ccd76..dfb0fb956 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1046,6 +1046,14 @@ def test_api_pair_history(botclient, ohlcv_history): assert rc.json()['data_stop'] == '2018-01-12 00:00:00+00:00' assert rc.json()['data_stop_ts'] == 1515715200000 + # No data found + rc = client_get(client, + f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" + "&timerange=20200111-20200112&strategy=DefaultStrategy") + assert_response(rc, 502) + assert rc.json()['error'] == ("Error querying /api/v1/pair_history: " + "No data for UNITTEST/BTC, 5m in 20200111-20200112 found.") + def test_api_plot_config(botclient): ftbot, client = botclient From c9e477214fddc6f60a08c3914c0fdd33ffcf9ecc Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Jan 2021 16:37:09 +0100 Subject: [PATCH 097/183] Allow protections to be set in the strategy --- docs/configuration.md | 3 +- docs/includes/protections.md | 47 +++++++++++++++++++++++- freqtrade/resolvers/strategy_resolver.py | 1 + freqtrade/strategy/interface.py | 3 ++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b70a85c04..13c7eb47b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -91,7 +91,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean | `pairlists` | Define one or more pairlists to be used. [More information below](#pairlists-and-pairlist-handlers).
*Defaults to `StaticPairList`.*
**Datatype:** List of Dicts -| `protections` | Define one or more protections to be used. [More information below](#protections).
**Datatype:** List of Dicts +| `protections` | Define one or more protections to be used. [More information below](#protections). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** List of Dicts | `telegram.enabled` | Enable the usage of Telegram.
**Datatype:** Boolean | `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String @@ -141,6 +141,7 @@ Values set in the configuration file always overwrite values set in the strategy * `stake_amount` * `unfilledtimeout` * `disable_dataframe_checks` +* `protections` * `use_sell_signal` (ask_strategy) * `sell_profit_only` (ask_strategy) * `ignore_roi_if_buy_signal` (ask_strategy) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index b98a0d662..8465392a4 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -7,7 +7,8 @@ Protections will protect your strategy from unexpected events and market conditi All protection end times are rounded up to the next candle to avoid sudden, unexpected intra-candle buys. !!! Note - Not all Protections will work for all strategies, and parameters will need to be tuned for your strategy to improve performance. + Not all Protections will work for all strategies, and parameters will need to be tuned for your strategy to improve performance. + To align your protection with your strategy, you can define protections in the strategy. !!! Tip Each Protection can be configured multiple times with different parameters, to allow different levels of protection (short-term / long-term). @@ -167,3 +168,47 @@ The below example assumes a timeframe of 1 hour: } ], ``` + +You can use the same in your strategy, the syntax is only slightly different: + +``` python +from freqtrade.strategy import IStrategy + +class AwesomeStrategy(IStrategy) + timeframe = '1h' + protections = [ + { + "method": "CooldownPeriod", + "stop_duration_candles": 5 + }, + { + "method": "MaxDrawdown", + "lookback_period_candles": 48, + "trade_limit": 20, + "stop_duration_candles": 4, + "max_allowed_drawdown": 0.2 + }, + { + "method": "StoplossGuard", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 2, + "only_per_pair": False + }, + { + "method": "LowProfitPairs", + "lookback_period_candles": 6, + "trade_limit": 2, + "stop_duration_candles": 60, + "required_profit": 0.02 + }, + { + "method": "LowProfitPairs", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 2, + "required_profit": 0.01 + } + ] + # ... +``` diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 73af00fee..2b7a4f0c2 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -73,6 +73,7 @@ class StrategyResolver(IResolver): ("order_time_in_force", None, None), ("stake_currency", None, None), ("stake_amount", None, None), + ("protections", None, None), ("startup_candle_count", None, None), ("unfilledtimeout", None, None), ("use_sell_signal", True, 'ask_strategy'), diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 1e4fc8b12..348e6a446 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -119,6 +119,9 @@ class IStrategy(ABC): # Count of candles the strategy requires before producing valid signals startup_candle_count: int = 0 + # Protections + protections: List + # Class level variables (intentional) containing # the dataprovider (dp) (access to other candles, historic data, ...) # and wallets - access to the current balance. From b43ef474ad231c1ac9cf87c760107ba7a5292a04 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Jan 2021 07:51:49 +0100 Subject: [PATCH 098/183] Fix expired candle implementation Improve and simplify test by passing the current time to the function --- freqtrade/strategy/interface.py | 8 ++++---- tests/strategy/test_interface.py | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index a18c6c915..b9d05f64f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -483,16 +483,16 @@ class IStrategy(ABC): latest['date'], pair, str(buy), str(sell)) timeframe_seconds = timeframe_to_seconds(timeframe) if self.ignore_expired_candle(latest_date=latest_date, + current_time=datetime.now(timezone.utc), timeframe_seconds=timeframe_seconds, buy=buy): return False, sell return buy, sell - def ignore_expired_candle(self, latest_date: datetime, timeframe_seconds: int, buy: bool): + def ignore_expired_candle(self, latest_date: datetime, current_time: datetime, + timeframe_seconds: int, buy: bool): if self.ignore_buying_expired_candle and buy: - current_time = datetime.now(timezone.utc) - timedelta( - seconds=self.ignore_buying_expired_candle_after) - time_delta = current_time - latest_date + timedelta(seconds=timeframe_seconds) + time_delta = current_time - (latest_date + timedelta(seconds=timeframe_seconds)) return time_delta.total_seconds() > self.ignore_buying_expired_candle_after else: return False diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index d7d113a4e..a3969d91b 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -106,21 +106,28 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history): assert log_has('Outdated history for pair xyz. Last tick is 16 minutes old', caplog) -def test_ignore_expired_candle(default_conf, ohlcv_history): +def test_ignore_expired_candle(default_conf): default_conf.update({'strategy': 'DefaultStrategy'}) strategy = StrategyResolver.load_strategy(default_conf) strategy.ignore_buying_expired_candle = True strategy.ignore_buying_expired_candle_after = 60 - ohlcv_history.loc[-1, 'date'] = arrow.utcnow() - # Take a copy to correctly modify the call - mocked_history = ohlcv_history.copy() - latest_date = mocked_history['date'].max() + latest_date = datetime(2020, 12, 30, 7, 0, 0, tzinfo=timezone.utc) + # Add 1 candle length as the "latest date" defines candle open. + current_time = latest_date + timedelta(seconds=80 + 300) assert strategy.ignore_expired_candle(latest_date=latest_date, + current_time=current_time, timeframe_seconds=300, buy=True) is True + current_time = latest_date + timedelta(seconds=30 + 300) + + assert not strategy.ignore_expired_candle(latest_date=latest_date, + current_time=current_time, + timeframe_seconds=300, + buy=True) is True + def test_assert_df_raise(mocker, caplog, ohlcv_history): ohlcv_history.loc[1, 'date'] = arrow.utcnow().shift(minutes=-16) From 9e66417e852783a4a1da4df0e11deae4d16fee86 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Jan 2021 19:21:42 +0100 Subject: [PATCH 099/183] Run CI for mac on 3.9 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daa10fea7..7b0418a11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: strategy: matrix: os: [ macos-latest ] - python-version: [3.7, 3.8] + python-version: [3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 @@ -146,7 +146,7 @@ jobs: run: | cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd .. - - name: Installation - *nix + - name: Installation - macOS run: | python -m pip install --upgrade pip export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH From 54ab61d18a717a3e00b57ae8acdbc60d698f37f9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Jan 2021 19:27:35 +0100 Subject: [PATCH 100/183] Install hdf5 via brew --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b0418a11..41ac0770d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,6 +148,7 @@ jobs: - name: Installation - macOS run: | + brew install hdf5 python -m pip install --upgrade pip export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH export TA_LIBRARY_PATH=${HOME}/dependencies/lib From 124cb5c5bff02a536da3529e05742fa353a87cb5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Jan 2021 19:36:50 +0100 Subject: [PATCH 101/183] Add cblosc brew dependency --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41ac0770d..961dfef71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,7 +148,7 @@ jobs: - name: Installation - macOS run: | - brew install hdf5 + brew install hdf5 c-blosc python -m pip install --upgrade pip export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH export TA_LIBRARY_PATH=${HOME}/dependencies/lib From bf182dc01e6187f81d3bd61dfcb4a43e754e1c5b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Jan 2021 20:03:34 +0100 Subject: [PATCH 102/183] Fix wrong key usage in trade_history_timebased --- freqtrade/exchange/exchange.py | 2 +- tests/conftest.py | 10 +++++----- tests/exchange/test_exchange.py | 25 +++++++++++++------------ 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 11a0ef8e6..b610b28f4 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -936,7 +936,7 @@ class Exchange: while True: t = await self._async_fetch_trades(pair, since=since) if len(t): - since = t[-1][1] + since = t[-1][0] trades.extend(t) # Reached the end of the defined-download period if until and t[-1][0] > until: diff --git a/tests/conftest.py b/tests/conftest.py index 9eda0e973..75a98dcc5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1492,11 +1492,11 @@ def trades_for_order(): @pytest.fixture(scope="function") def trades_history(): - return [[1565798389463, '126181329', None, 'buy', 0.019627, 0.04, 0.00078508], - [1565798399629, '126181330', None, 'buy', 0.019627, 0.244, 0.004788987999999999], - [1565798399752, '126181331', None, 'sell', 0.019626, 0.011, 0.00021588599999999999], - [1565798399862, '126181332', None, 'sell', 0.019626, 0.011, 0.00021588599999999999], - [1565798399872, '126181333', None, 'sell', 0.019626, 0.011, 0.00021588599999999999]] + return [[1565798389463, '12618132aa9', None, 'buy', 0.019627, 0.04, 0.00078508], + [1565798399629, '1261813bb30', None, 'buy', 0.019627, 0.244, 0.004788987999999999], + [1565798399752, '1261813cc31', None, 'sell', 0.019626, 0.011, 0.00021588599999999999], + [1565798399862, '126181cc332', None, 'sell', 0.019626, 0.011, 0.00021588599999999999], + [1565798399872, '1261aa81333', None, 'sell', 0.019626, 0.011, 0.00021588599999999999]] @pytest.fixture(scope="function") diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a42ff52e4..f33382da4 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1759,36 +1759,37 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang @pytest.mark.asyncio @pytest.mark.parametrize("exchange_name", EXCHANGES) async def test__async_get_trade_history_time(default_conf, mocker, caplog, exchange_name, - trades_history): + fetch_trades_result): caplog.set_level(logging.DEBUG) async def mock_get_trade_hist(pair, *args, **kwargs): - if kwargs['since'] == trades_history[0][0]: - return trades_history[:-1] + if kwargs['since'] == fetch_trades_result[0]['timestamp']: + return fetch_trades_result[:-1] else: - return trades_history[-1:] + return fetch_trades_result[-1:] caplog.set_level(logging.DEBUG) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) # Monkey-patch async function - exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist) + exchange._api_async.fetch_trades = MagicMock(side_effect=mock_get_trade_hist) pair = 'ETH/BTC' - ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0][0], - until=trades_history[-1][0]-1) + ret = await exchange._async_get_trade_history_time(pair, + since=fetch_trades_result[0]['timestamp'], + until=fetch_trades_result[-1]['timestamp']-1) assert type(ret) is tuple assert ret[0] == pair assert type(ret[1]) is list - assert len(ret[1]) == len(trades_history) - assert exchange._async_fetch_trades.call_count == 2 - fetch_trades_cal = exchange._async_fetch_trades.call_args_list + assert len(ret[1]) == len(fetch_trades_result) + assert exchange._api_async.fetch_trades.call_count == 2 + fetch_trades_cal = exchange._api_async.fetch_trades.call_args_list # first call (using since, not fromId) assert fetch_trades_cal[0][0][0] == pair - assert fetch_trades_cal[0][1]['since'] == trades_history[0][0] + assert fetch_trades_cal[0][1]['since'] == fetch_trades_result[0]['timestamp'] # 2nd call assert fetch_trades_cal[1][0][0] == pair - assert fetch_trades_cal[0][1]['since'] == trades_history[0][0] + assert fetch_trades_cal[1][1]['since'] == fetch_trades_result[-2]['timestamp'] assert log_has_re(r"Stopping because until was reached.*", caplog) From 4f126bea3582242155f412a5e4175b8cfe363694 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Jan 2021 20:06:26 +0100 Subject: [PATCH 103/183] Change trades-test2 to better test correct behaviour --- tests/exchange/test_exchange.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index f33382da4..7d9954cb9 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1718,8 +1718,8 @@ async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name, @pytest.mark.asyncio @pytest.mark.parametrize("exchange_name", EXCHANGES) -async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchange_name, - trades_history): +async def test__async_get_trade_history_id(default_conf, mocker, exchange_name, + fetch_trades_result): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) pagination_arg = exchange._trades_pagination_arg @@ -1727,28 +1727,29 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang async def mock_get_trade_hist(pair, *args, **kwargs): if 'since' in kwargs: # Return first 3 - return trades_history[:-2] - elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3][1]: + return fetch_trades_result[:-2] + elif kwargs.get('params', {}).get(pagination_arg) == fetch_trades_result[-3]['id']: # Return 2 - return trades_history[-3:-1] + return fetch_trades_result[-3:-1] else: # Return last 2 - return trades_history[-2:] + return fetch_trades_result[-2:] # Monkey-patch async function - exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist) + exchange._api_async.fetch_trades = MagicMock(side_effect=mock_get_trade_hist) pair = 'ETH/BTC' - ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0][0], - until=trades_history[-1][0]-1) + ret = await exchange._async_get_trade_history_id(pair, + since=fetch_trades_result[0]['timestamp'], + until=fetch_trades_result[-1]['timestamp'] - 1) assert type(ret) is tuple assert ret[0] == pair assert type(ret[1]) is list - assert len(ret[1]) == len(trades_history) - assert exchange._async_fetch_trades.call_count == 3 - fetch_trades_cal = exchange._async_fetch_trades.call_args_list + assert len(ret[1]) == len(fetch_trades_result) + assert exchange._api_async.fetch_trades.call_count == 3 + fetch_trades_cal = exchange._api_async.fetch_trades.call_args_list # first call (using since, not fromId) assert fetch_trades_cal[0][0][0] == pair - assert fetch_trades_cal[0][1]['since'] == trades_history[0][0] + assert fetch_trades_cal[0][1]['since'] == fetch_trades_result[0]['timestamp'] # 2nd call assert fetch_trades_cal[1][0][0] == pair From 2e7faa782c7980d0a49d56ebb8c8d08399e0100c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Jan 2021 06:51:37 +0100 Subject: [PATCH 104/183] Add documentation section for macOS installation error on 3.999999999 --- docs/installation.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/installation.md b/docs/installation.md index 73e791c56..a23399441 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -245,6 +245,19 @@ open /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10 If this file is inexistent, then you're probably on a different version of MacOS, so you may need to consult the internet for specific resolution details. +### MacOS installation error with python 3.9 + +When using python 3.9 on macOS, it's currently necessary to install some os-level modules to allow dependencies to compile. +The errors you'll see happen during installation and are related to the installation of `tables` or `blosc`. + +You can install the necessary libraries with the following command: + +``` bash +brew install hdf5 c-blosc +``` + +After this, please run the installation (script) again. + ----- Now you have an environment ready, the next step is From bd5f46e4c22fb0e8025e7c0bc0f5493e260c97d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Jan 2021 07:16:36 +0000 Subject: [PATCH 105/183] Bump pytest-mock from 3.4.0 to 3.5.0 Bumps [pytest-mock](https://github.com/pytest-dev/pytest-mock) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/pytest-dev/pytest-mock/releases) - [Changelog](https://github.com/pytest-dev/pytest-mock/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-mock/compare/v3.4.0...v3.5.0) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e918a9b90..3c037b7cd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,7 @@ mypy==0.790 pytest==6.2.1 pytest-asyncio==0.14.0 pytest-cov==2.10.1 -pytest-mock==3.4.0 +pytest-mock==3.5.0 pytest-random-order==1.0.4 isort==5.7.0 From f3319e1382d26c1685339e2def3419435ec2b9eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Jan 2021 07:16:48 +0000 Subject: [PATCH 106/183] Bump prompt-toolkit from 3.0.8 to 3.0.9 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.8 to 3.0.9. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/commits/3.0.9) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5099491dd..e7d8398eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,4 +36,4 @@ pyjwt==1.7.1 colorama==0.4.4 # Building config files interactively questionary==1.9.0 -prompt-toolkit==3.0.8 +prompt-toolkit==3.0.9 From 784630e2f2a215f7e811e0e393564891f6d45233 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Jan 2021 07:16:49 +0000 Subject: [PATCH 107/183] Bump uvicorn from 0.13.2 to 0.13.3 Bumps [uvicorn](https://github.com/encode/uvicorn) from 0.13.2 to 0.13.3. - [Release notes](https://github.com/encode/uvicorn/releases) - [Changelog](https://github.com/encode/uvicorn/blob/master/CHANGELOG.md) - [Commits](https://github.com/encode/uvicorn/compare/0.13.2...0.13.3) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5099491dd..44fadce73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ sdnotify==0.3.2 # API Server fastapi==0.63.0 -uvicorn==0.13.2 +uvicorn==0.13.3 pyjwt==1.7.1 # Support for colorized terminal output From 3cf506fa5d2b9dc8f764b83fbf3921fdea508af1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Jan 2021 07:16:49 +0000 Subject: [PATCH 108/183] Bump ccxt from 1.40.14 to 1.40.25 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.40.14 to 1.40.25. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst) - [Commits](https://github.com/ccxt/ccxt/compare/1.40.14...1.40.25) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5099491dd..c8d2d0d54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.19.4 pandas==1.2.0 -ccxt==1.40.14 +ccxt==1.40.25 aiohttp==3.7.3 SQLAlchemy==1.3.22 python-telegram-bot==13.1 From 4d2c59b7ecbb9b8431da1c922614480cc91c91b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Jan 2021 07:47:47 +0000 Subject: [PATCH 109/183] Bump numpy from 1.19.4 to 1.19.5 Bumps [numpy](https://github.com/numpy/numpy) from 1.19.4 to 1.19.5. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/master/doc/HOWTO_RELEASE.rst.txt) - [Commits](https://github.com/numpy/numpy/compare/v1.19.4...v1.19.5) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 87ec29fb6..b564a4244 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.19.4 +numpy==1.19.5 pandas==1.2.0 ccxt==1.40.25 From c8df3c4730dacd955ed2bb207fd02b099b4c007e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Jan 2021 07:48:31 +0000 Subject: [PATCH 110/183] Bump pyjwt from 1.7.1 to 2.0.0 Bumps [pyjwt](https://github.com/jpadilla/pyjwt) from 1.7.1 to 2.0.0. - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.md) - [Commits](https://github.com/jpadilla/pyjwt/compare/1.7.1...2.0.0) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 87ec29fb6..ae6ba2e46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ sdnotify==0.3.2 # API Server fastapi==0.63.0 uvicorn==0.13.3 -pyjwt==1.7.1 +pyjwt==2.0.0 # Support for colorized terminal output colorama==0.4.4 From 378a252ad1d3a40e531ce1dcadf6ac8d4fcb1653 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Jan 2021 13:46:43 +0100 Subject: [PATCH 111/183] Fix #4161 - by not using the problematic method for windows --- tests/optimize/test_optimize_reports.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index a0e1932ff..e194e7de4 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -1,5 +1,5 @@ import re -from datetime import timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path import pandas as pd @@ -121,8 +121,8 @@ def test_generate_backtest_stats(default_conf, testdatadir): } assert strat_stats['max_drawdown'] == 0.0 - assert strat_stats['drawdown_start'] == Arrow.fromtimestamp(0).datetime - assert strat_stats['drawdown_end'] == Arrow.fromtimestamp(0).datetime + assert strat_stats['drawdown_start'] == datetime(1970, 1, 1, tzinfo=timezone.utc) + assert strat_stats['drawdown_end'] == datetime(1970, 1, 1, tzinfo=timezone.utc) assert strat_stats['drawdown_end_ts'] == 0 assert strat_stats['drawdown_start_ts'] == 0 assert strat_stats['pairlist'] == ['UNITTEST/BTC'] From 8631a54514084879c9507fe7d0f89ae4464d84b5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Jan 2021 19:27:51 +0100 Subject: [PATCH 112/183] Fix test due to pyjwt2.0 --- freqtrade/rpc/api_server/api_auth.py | 2 +- tests/rpc/test_rpc_apiserver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_auth.py b/freqtrade/rpc/api_server/api_auth.py index 110bb2a25..a39e31b85 100644 --- a/freqtrade/rpc/api_server/api_auth.py +++ b/freqtrade/rpc/api_server/api_auth.py @@ -44,7 +44,7 @@ def get_user_from_token(token, secret_key: str, token_type: str = "access"): return username -def create_token(data: dict, secret_key: str, token_type: str = "access") -> bytes: +def create_token(data: dict, secret_key: str, token_type: str = "access") -> str: to_encode = data.copy() if token_type == "access": expire = datetime.utcnow() + timedelta(minutes=15) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index dfb0fb956..2212d4a79 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -93,7 +93,7 @@ def test_api_auth(): create_token({'identity': {'u': 'Freqtrade'}}, 'secret1234', token_type="NotATokenType") token = create_token({'identity': {'u': 'Freqtrade'}}, 'secret1234') - assert isinstance(token, bytes) + assert isinstance(token, str) u = get_user_from_token(token, 'secret1234') assert u == 'Freqtrade' From ddecf3ef98b4c06a056effa319eaf8097d0ce190 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Jan 2021 05:38:34 +0000 Subject: [PATCH 113/183] Bump mkdocs-material from 6.2.3 to 6.2.4 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 6.2.3 to 6.2.4. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/6.2.3...6.2.4) Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 2db336f4a..adf4bc96c 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,3 +1,3 @@ -mkdocs-material==6.2.3 +mkdocs-material==6.2.4 mdx_truly_sane_lists==1.2 pymdown-extensions==8.1 From a34753fcb16db610b7fcf3717d372c35b0d743c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Jan 2021 05:38:45 +0000 Subject: [PATCH 114/183] Bump prompt-toolkit from 3.0.9 to 3.0.10 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.9 to 3.0.10. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.9...3.0.10) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1d0d57091..69081108a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,4 +36,4 @@ pyjwt==2.0.0 colorama==0.4.4 # Building config files interactively questionary==1.9.0 -prompt-toolkit==3.0.9 +prompt-toolkit==3.0.10 From f1809286cfefd216a43ceac6e35d113f22b26a1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Jan 2021 05:38:47 +0000 Subject: [PATCH 115/183] Bump ccxt from 1.40.25 to 1.40.30 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.40.25 to 1.40.30. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst) - [Commits](https://github.com/ccxt/ccxt/compare/1.40.25...1.40.30) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1d0d57091..fc3fb1a00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.19.5 pandas==1.2.0 -ccxt==1.40.25 +ccxt==1.40.30 aiohttp==3.7.3 SQLAlchemy==1.3.22 python-telegram-bot==13.1 From 59efc5f083050bc34e3ed5cdf51004b286d92124 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Jan 2021 05:38:47 +0000 Subject: [PATCH 116/183] Bump pytest-mock from 3.5.0 to 3.5.1 Bumps [pytest-mock](https://github.com/pytest-dev/pytest-mock) from 3.5.0 to 3.5.1. - [Release notes](https://github.com/pytest-dev/pytest-mock/releases) - [Changelog](https://github.com/pytest-dev/pytest-mock/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-mock/compare/v3.5.0...v3.5.1) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3c037b7cd..883c3089f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,7 @@ mypy==0.790 pytest==6.2.1 pytest-asyncio==0.14.0 pytest-cov==2.10.1 -pytest-mock==3.5.0 +pytest-mock==3.5.1 pytest-random-order==1.0.4 isort==5.7.0 From f159c464380ba12d283cd1bddddc661a3cff3b32 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 11 Jan 2021 07:55:01 +0100 Subject: [PATCH 117/183] Include stoploss_on_exchange in stoploss_guard fix #4183 --- docs/includes/protections.md | 2 +- freqtrade/plugins/protections/stoploss_guard.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 8465392a4..95fa53ad2 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -58,7 +58,7 @@ The below example stops trading for all pairs for 4 candles after the last trade ``` !!! Note - `StoplossGuard` considers all trades with the results `"stop_loss"` and `"trailing_stop_loss"` if the resulting profit was negative. + `StoplossGuard` considers all trades with the results `"stop_loss"`, `"stoploss_on_exchange"` and `"trailing_stop_loss"` if the resulting profit was negative. `trade_limit` and `lookback_period` will need to be tuned for your strategy. #### MaxDrawdown diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 193907ddc..92fae54cb 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -53,8 +53,9 @@ class StoplossGuard(IProtection): # trades = Trade.get_trades(filters).all() trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) - trades = [trade for trade in trades1 if str(trade.sell_reason) == SellType.STOP_LOSS.value - or (str(trade.sell_reason) == SellType.TRAILING_STOP_LOSS.value + trades = [trade for trade in trades1 if (str(trade.sell_reason) in ( + SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value, + SellType.STOPLOSS_ON_EXCHANGE.value) and trade.close_profit < 0)] if len(trades) > self._trade_limit: From dbc25f00acdded97725d1e551ea1848665097e3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 11 Jan 2021 19:12:03 +0100 Subject: [PATCH 118/183] Switch full config from bittrex to binance bittrex no longer supports volumepairlist. closes #4192 --- config_full.json.example | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 7cdd6af67..db8debb2c 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -103,7 +103,7 @@ } ], "exchange": { - "name": "bittrex", + "name": "binance", "sandbox": false, "key": "your_exchange_key", "secret": "your_exchange_secret", @@ -115,16 +115,21 @@ "aiohttp_trust_env": false }, "pair_whitelist": [ + "ALGO/BTC", + "ATOM/BTC", + "BAT/BTC", + "BCH/BTC", + "BRD/BTC", + "EOS/BTC", "ETH/BTC", + "IOTA/BTC", + "LINK/BTC", "LTC/BTC", - "ETC/BTC", - "DASH/BTC", - "ZEC/BTC", - "XLM/BTC", - "NXT/BTC", - "TRX/BTC", - "ADA/BTC", - "XMR/BTC" + "NEO/BTC", + "NXS/BTC", + "XMR/BTC", + "XRP/BTC", + "XTZ/BTC" ], "pair_blacklist": [ "DOGE/BTC" @@ -147,7 +152,7 @@ "remove_pumps": false }, "telegram": { - "enabled": true, + "enabled": false, "token": "your_telegram_token", "chat_id": "your_telegram_chat_id", "notification_settings": { From 63a579dbab0eaafa0c699f01e6d266729575c5fa Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 11 Jan 2021 19:30:07 +0100 Subject: [PATCH 119/183] Add sell_profit_offset parameter Allows defining positive offsets before enabling the sell signal --- config_full.json.example | 1 + docs/configuration.md | 3 ++- freqtrade/constants.py | 1 + freqtrade/optimize/optimize_reports.py | 1 + freqtrade/resolvers/strategy_resolver.py | 1 + freqtrade/strategy/interface.py | 9 +++++---- 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index db8debb2c..ef791d267 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -42,6 +42,7 @@ "order_book_max": 1, "use_sell_signal": true, "sell_profit_only": false, + "sell_profit_offset": 0.0, "ignore_roi_if_buy_signal": false }, "order_types": { diff --git a/docs/configuration.md b/docs/configuration.md index 13c7eb47b..926506f22 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -72,7 +72,8 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `ask_strategy.order_book_min` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
**Datatype:** Positive Integer | `ask_strategy.order_book_max` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
**Datatype:** Positive Integer | `ask_strategy.use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
**Datatype:** Boolean -| `ask_strategy.sell_profit_only` | Wait until the bot makes a positive profit before taking a sell decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `ask_strategy.sell_profit_only` | Wait until the bot reaches `ask_strategy.sell_profit_offset` before taking a sell decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `ask_strategy.sell_profit_offset` | Sell-signal is only active above this value. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0`.*
**Datatype:** Float (as ratio) | `ask_strategy.ignore_roi_if_buy_signal` | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `order_types` | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict | `order_time_in_force` | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e7d7e80f6..d48ab635e 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -154,6 +154,7 @@ CONF_SCHEMA = { 'order_book_max': {'type': 'integer', 'minimum': 1, 'maximum': 50}, 'use_sell_signal': {'type': 'boolean'}, 'sell_profit_only': {'type': 'boolean'}, + 'sell_profit_offset': {'type': 'number', 'minimum': 0.0}, 'ignore_roi_if_buy_signal': {'type': 'boolean'} } }, diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index d029ecd13..6c70b7c84 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -299,6 +299,7 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], 'minimal_roi': config['minimal_roi'], 'use_sell_signal': config['ask_strategy']['use_sell_signal'], 'sell_profit_only': config['ask_strategy']['sell_profit_only'], + 'sell_profit_offset': config['ask_strategy']['sell_profit_offset'], 'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'], **daily_stats, } diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 2b7a4f0c2..825e6572a 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -79,6 +79,7 @@ class StrategyResolver(IResolver): ("use_sell_signal", True, 'ask_strategy'), ("sell_profit_only", False, 'ask_strategy'), ("ignore_roi_if_buy_signal", False, 'ask_strategy'), + ("sell_profit_offset", 0.0, 'ask_strategy'), ("disable_dataframe_checks", False, None), ] for attribute, default, subkey in attributes: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 348e6a446..8546b1eda 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -505,18 +505,19 @@ class IStrategy(ABC): # Set current rate to high for backtesting sell current_rate = high or rate current_profit = trade.calc_profit_ratio(current_rate) - config_ask_strategy = self.config.get('ask_strategy', {}) + ask_strategy = self.config.get('ask_strategy', {}) # if buy signal and ignore_roi is set, we don't need to evaluate min_roi. - roi_reached = (not (buy and config_ask_strategy.get('ignore_roi_if_buy_signal', False)) + roi_reached = (not (buy and ask_strategy.get('ignore_roi_if_buy_signal', False)) and self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date)) - if config_ask_strategy.get('sell_profit_only', False) and trade.calc_profit(rate=rate) <= 0: + if (ask_strategy.get('sell_profit_only', False) + and trade.calc_profit(rate=rate) <= ask_strategy.get('sell_profit_offset', 0)): # Negative profits and sell_profit_only - ignore sell signal sell_signal = False else: - sell_signal = sell and not buy and config_ask_strategy.get('use_sell_signal', True) + sell_signal = sell and not buy and ask_strategy.get('use_sell_signal', True) # TODO: return here if sell-signal should be favored over ROI # Start evaluations From b062b836cc6e5c12e3eebf964d191fe70e1d04ae Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 11 Jan 2021 19:42:44 +0100 Subject: [PATCH 120/183] Add test for sell_profit_offset --- tests/test_freqtradebot.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 5c5666788..e6aff3352 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3065,6 +3065,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, limit_buy default_conf['ask_strategy'] = { 'use_sell_signal': True, 'sell_profit_only': True, + 'sell_profit_offset': 0.1, } freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -3076,7 +3077,11 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, limit_buy trade.update(limit_buy_order) freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True)) + assert freqtrade.handle_trade(trade) is False + + freqtrade.config['ask_strategy']['sell_profit_offset'] = 0.0 assert freqtrade.handle_trade(trade) is True + assert trade.sell_reason == SellType.SELL_SIGNAL.value From 4d7ffa8c810e20b95ca746e61b5e7f2e964ac5ad Mon Sep 17 00:00:00 2001 From: nas- Date: Tue, 12 Jan 2021 01:13:58 +0100 Subject: [PATCH 121/183] Added suppoort for regex in whitelist --- freqtrade/commands/data_commands.py | 17 +++--- freqtrade/edge/edge_positioning.py | 7 ++- freqtrade/exchange/exchange.py | 4 +- freqtrade/plot/plotting.py | 14 +++-- freqtrade/plugins/pairlist/IPairList.py | 9 +++ freqtrade/plugins/pairlist/StaticPairList.py | 5 +- freqtrade/plugins/pairlistmanager.py | 20 +++++++ tests/exchange/test_exchange.py | 63 ++++++++++---------- tests/test_plotting.py | 5 +- 9 files changed, 93 insertions(+), 51 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 25c7d0436..1ce02eee5 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -10,6 +10,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_oh refresh_backtest_trades_data) from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode @@ -42,15 +43,17 @@ def start_download_data(args: Dict[str, Any]) -> None: "Downloading data requires a list of pairs. " "Please check the documentation on how to configure this.") - logger.info(f"About to download pairs: {config['pairs']}, " - f"intervals: {config['timeframes']} to {config['datadir']}") - pairs_not_available: List[str] = [] # Init exchange exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) # Manual validations of relevant settings exchange.validate_pairs(config['pairs']) + expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets)) + + logger.info(f"About to download pairs: {expanded_pairs}, " + f"intervals: {config['timeframes']} to {config['datadir']}") + for timeframe in config['timeframes']: exchange.validate_timeframes(timeframe) @@ -58,20 +61,20 @@ def start_download_data(args: Dict[str, Any]) -> None: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( - exchange, pairs=config['pairs'], datadir=config['datadir'], + exchange, pairs=expanded_pairs, datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format=config['dataformat_trades']) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( - pairs=config['pairs'], timeframes=config['timeframes'], + pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], - ) + ) else: pairs_not_available = refresh_backtest_ohlcv_data( - exchange, pairs=config['pairs'], timeframes=config['timeframes'], + exchange, pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv']) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 037717c68..e549a3701 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -12,6 +12,7 @@ from freqtrade.configuration import TimeRange from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT from freqtrade.data.history import get_timerange, load_data, refresh_data from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.strategy.interface import SellType @@ -80,10 +81,12 @@ class Edge: if config.get('fee'): self.fee = config['fee'] else: - self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0]) + self.fee = self.exchange.get_fee(symbol=expand_pairlist( + self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0]) def calculate(self) -> bool: - pairs = self.config['exchange']['pair_whitelist'] + pairs = expand_pairlist(self.config['exchange']['pair_whitelist'], + list(self.exchange.markets)) heartbeat = self.edge_config.get('process_throttle_secs') if (self._last_updated > 0) and ( diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b610b28f4..489f70528 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -25,6 +25,7 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier, retrier_async) from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist CcxtModuleType = Any @@ -335,8 +336,9 @@ class Exchange: if not self.markets: logger.warning('Unable to validate pairs (assuming they are correct).') return + extended_pairs = expand_pairlist(pairs, list(self.markets)) invalid_pairs = [] - for pair in pairs: + for pair in extended_pairs: # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs # TODO: add a support for having coins in BTC/USDT format if self.markets and pair not in self.markets: diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 40e3da9c9..996c5276c 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -13,6 +13,7 @@ from freqtrade.data.history import get_timerange, load_data from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds from freqtrade.misc import pair_to_filename +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.strategy import IStrategy @@ -29,16 +30,16 @@ except ImportError: exit(1) -def init_plotscript(config, startup_candles: int = 0): +def init_plotscript(config, markets: List, startup_candles: int = 0): """ Initialize objects needed for plotting :return: Dict with candle (OHLCV) data, trades and pairs """ if "pairs" in config: - pairs = config['pairs'] + pairs = expand_pairlist(config['pairs'], markets) else: - pairs = config['exchange']['pair_whitelist'] + pairs = expand_pairlist(config['exchange']['pair_whitelist'], markets) # Set timerange to use timerange = TimeRange.parse_timerange(config.get('timerange')) @@ -177,7 +178,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: trades['desc'] = trades.apply(lambda row: f"{round(row['profit_percent'] * 100, 1)}%, " f"{row['sell_reason']}, " f"{row['trade_duration']} min", - axis=1) + axis=1) trade_buys = go.Scatter( x=trades["open_date"], y=trades["open_rate"], @@ -527,7 +528,7 @@ def load_and_plot_trades(config: Dict[str, Any]): exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) IStrategy.dp = DataProvider(config, exchange) - plot_elements = init_plotscript(config, strategy.startup_candle_count) + plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count) timerange = plot_elements['timerange'] trades = plot_elements['trades'] pair_counter = 0 @@ -562,7 +563,8 @@ def plot_profit(config: Dict[str, Any]) -> None: But should be somewhat proportional, and therefor useful in helping out to find a good algorithm. """ - plot_elements = init_plotscript(config) + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) + plot_elements = init_plotscript(config, list(exchange.markets)) trades = plot_elements['trades'] # Filter trades to relevant pairs # Remove open pairs - we don't know the profit yet so can't calculate profit for these. diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 865aa90d6..f9809aeb1 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -124,6 +124,15 @@ class IPairList(LoggingMixin, ABC): """ return self._pairlistmanager.verify_blacklist(pairlist, logmethod) + def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]: + """ + Proxy method to verify_whitelist for easy access for child classes. + :param pairlist: Pairlist to validate + :param logmethod: Function that'll be called, `logger.info` or `logger.warning`. + :return: pairlist - whitelisted pairs + """ + return self._pairlistmanager.verify_whitelist(pairlist, logmethod) + def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index dd592e0ca..c216a322d 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -50,9 +50,10 @@ class StaticPairList(IPairList): :return: List of pairs """ if self._allow_inactive: - return self._config['exchange']['pair_whitelist'] + return self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info) else: - return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) + return self._whitelist_for_active_markets( + self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info)) def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index ad7b46cb8..c471d7c51 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -59,6 +59,11 @@ class PairListManager(): """The expanded blacklist (including wildcard expansion)""" return expand_pairlist(self._blacklist, self._exchange.get_markets().keys()) + @property + def expanded_whitelist(self) -> List[str]: + """The expanded whitelist (including wildcard expansion)""" + return expand_pairlist(self._whitelist, self._exchange.get_markets().keys()) + @property def name_list(self) -> List[str]: """Get list of loaded Pairlist Handler names""" @@ -129,6 +134,21 @@ class PairListManager(): pairlist.remove(pair) return pairlist + def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]: + """ + Verify and remove items from pairlist - returning a filtered pairlist. + Logs a warning or info depending on `aswarning`. + Pairlist Handlers explicitly using this method shall use + `logmethod=logger.info` to avoid spamming with warning messages + :return: pairlist - blacklisted pairs + """ + try: + whitelist = self.expanded_whitelist + except ValueError as err: + logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") + return [] + return whitelist + def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes: """ Create list of pair tuples with (pair, timeframe) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 7d9954cb9..8cd2a9bca 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -505,37 +505,38 @@ def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs d Exchange(default_conf) -def test_validate_pairs_not_available(default_conf, mocker): - api_mock = MagicMock() - type(api_mock).markets = PropertyMock(return_value={ - 'XRP/BTC': {'inactive': True} - }) - mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') - - with pytest.raises(OperationalException, match=r'not available'): - Exchange(default_conf) - - -def test_validate_pairs_exception(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - api_mock = MagicMock() - mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) - - type(api_mock).markets = PropertyMock(return_value={}) - mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') - mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') - mocker.patch('freqtrade.exchange.Exchange._load_async_markets') - - with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): - Exchange(default_conf) - - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})) - Exchange(default_conf) - assert log_has('Unable to validate pairs (assuming they are correct).', caplog) +# This cannot happen anymore as expand_pairlist implicitly filters out unavaliablie pairs +# def test_validate_pairs_not_available(default_conf, mocker): +# api_mock = MagicMock() +# type(api_mock).markets = PropertyMock(return_value={ +# 'XRP/BTC': {'inactive': True, 'base': 'XRP', 'quote': 'BTC'} +# }) +# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) +# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') +# mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') +# mocker.patch('freqtrade.exchange.Exchange._load_async_markets') +# +# with pytest.raises(OperationalException, match=r'not available'): +# Exchange(default_conf) +# +# +# def test_validate_pairs_exception(default_conf, mocker, caplog): +# caplog.set_level(logging.INFO) +# api_mock = MagicMock() +# mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) +# +# type(api_mock).markets = PropertyMock(return_value={}) +# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) +# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') +# mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') +# mocker.patch('freqtrade.exchange.Exchange._load_async_markets') +# +# with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): +# Exchange(default_conf) +# +# mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})) +# Exchange(default_conf) +# assert log_has('Unable to validate pairs (assuming they are correct).', caplog) def test_validate_pairs_restricted(default_conf, mocker, caplog): diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 8e7b0ef7c..8f3ac464e 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -47,14 +47,15 @@ def test_init_plotscript(default_conf, mocker, testdatadir): default_conf['timeframe'] = "5m" default_conf["datadir"] = testdatadir default_conf['exportfilename'] = testdatadir / "backtest-result_test.json" - ret = init_plotscript(default_conf) + supported_markets = ["TRX/BTC", "ADA/BTC"] + ret = init_plotscript(default_conf, supported_markets) assert "ohlcv" in ret assert "trades" in ret assert "pairs" in ret assert 'timerange' in ret default_conf['pairs'] = ["TRX/BTC", "ADA/BTC"] - ret = init_plotscript(default_conf, 20) + ret = init_plotscript(default_conf, supported_markets, 20) assert "ohlcv" in ret assert "TRX/BTC" in ret["ohlcv"] assert "ADA/BTC" in ret["ohlcv"] From e328182bd7d3bd40f850bb26179f97e868786ada Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Tue, 12 Jan 2021 07:30:39 +0100 Subject: [PATCH 122/183] Changed workings so it only needs to timing-parameter, instead of also requiring a boolean value --- docs/configuration.md | 7 ++----- freqtrade/resolvers/strategy_resolver.py | 1 - freqtrade/strategy/interface.py | 8 ++++---- tests/strategy/test_interface.py | 1 - 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6ae37e7b3..93693c919 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -74,8 +74,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `ask_strategy.use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
**Datatype:** Boolean | `ask_strategy.sell_profit_only` | Wait until the bot makes a positive profit before taking a sell decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `ask_strategy.ignore_roi_if_buy_signal` | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean -| `ask_strategy.ignore_buying_expired_candle` | Enables usage of skipping buys on candles that are older than a specified period.
*Defaults to `False`*
**Datatype:** Boolean -| `ask_strategy.ignore_buying_expired_candle_after` | Specifies the number of seconds until a buy signal is no longer used when setting `ignore_buying_expired_candle`.
**Datatype:** Integer +| `ask_strategy.ignore_buying_expired_candle_after` | Specifies the number of seconds until a buy signal is no longer used.
**Datatype:** Integer | `order_types` | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict | `order_time_in_force` | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict | `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
**Datatype:** String @@ -146,7 +145,6 @@ Values set in the configuration file always overwrite values set in the strategy * `use_sell_signal` (ask_strategy) * `sell_profit_only` (ask_strategy) * `ignore_roi_if_buy_signal` (ask_strategy) -* `ignore_buying_expired_candle` (ask_strategy) * `ignore_buying_expired_candle_after` (ask_strategy) ### Configuring amount per trade @@ -679,13 +677,12 @@ freqtrade When working with larger timeframes (for example 1h or more) and using a low `max_open_trades` value, the last candle can be processed as soon as a trade slot becomes available. When processing the last candle, this can lead to a situation where it may not be desirable to use the buy signal on that candle. For example, when using a condition in your strategy where you use a cross-over, that point may have passed too long ago for you to start a trade on it. -In these situations, you can enable the functionality to ignore candles that are beyond a specified period by setting `ask_strategy.ignore_buying_expired_candle` to `true`. After this, you can set `ask_strategy.ignore_buying_expired_candle_after` to the number of seconds after which the candle becomes expired. +In these situations, you can enable the functionality to ignore candles that are beyond a specified period by setting `ask_strategy.ignore_buying_expired_candle_after` to a positive number, indicating the number of seconds after which the candle becomes expired. For example, if your strategy is using a 1h timeframe, and you only want to buy within the first 5 minutes when a new candle comes in, you can add the following configuration to your strategy: ``` jsonc "ask_strategy":{ - "ignore_buying_expired_candle" = true "ignore_buying_expired_candle_after" = 300 # 5 minutes "price_side": "bid", // ... diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 5872d95a6..3b7374326 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -79,7 +79,6 @@ class StrategyResolver(IResolver): ("sell_profit_only", False, 'ask_strategy'), ("ignore_roi_if_buy_signal", False, 'ask_strategy'), ("disable_dataframe_checks", False, None), - ("ignore_buying_expired_candle", None, 'ask_strategy'), ("ignore_buying_expired_candle_after", 0, 'ask_strategy') ] for attribute, default, subkey in attributes: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index b9d05f64f..57dcdeb3c 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -113,9 +113,7 @@ class IStrategy(ABC): # run "populate_indicators" only for new candle process_only_new_candles: bool = False - # Don't buy on expired candles - ignore_buying_expired_candle: bool = False - # Number of seconds after which the candle will no longer result in a buy + # Number of seconds after which the candle will no longer result in a buy on expired candles ignore_buying_expired_candle_after: int = 0 # Disable checking the dataframe (converts the error into a warning message) @@ -491,7 +489,9 @@ class IStrategy(ABC): def ignore_expired_candle(self, latest_date: datetime, current_time: datetime, timeframe_seconds: int, buy: bool): - if self.ignore_buying_expired_candle and buy: + if self.ignore_buying_expired_candle_after \ + and self.ignore_buying_expired_candle_after > 0\ + and buy: time_delta = current_time - (latest_date + timedelta(seconds=timeframe_seconds)) return time_delta.total_seconds() > self.ignore_buying_expired_candle_after else: diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index a3969d91b..f158a1518 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -109,7 +109,6 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history): def test_ignore_expired_candle(default_conf): default_conf.update({'strategy': 'DefaultStrategy'}) strategy = StrategyResolver.load_strategy(default_conf) - strategy.ignore_buying_expired_candle = True strategy.ignore_buying_expired_candle_after = 60 latest_date = datetime(2020, 12, 30, 7, 0, 0, tzinfo=timezone.utc) From 71f45021b9083862b6d06fe7adf425a7de73ff49 Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Tue, 12 Jan 2021 07:35:30 +0100 Subject: [PATCH 123/183] Removed redundant statement --- freqtrade/strategy/interface.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 57dcdeb3c..40debe78f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -490,7 +490,6 @@ class IStrategy(ABC): def ignore_expired_candle(self, latest_date: datetime, current_time: datetime, timeframe_seconds: int, buy: bool): if self.ignore_buying_expired_candle_after \ - and self.ignore_buying_expired_candle_after > 0\ and buy: time_delta = current_time - (latest_date + timedelta(seconds=timeframe_seconds)) return time_delta.total_seconds() > self.ignore_buying_expired_candle_after From 1f6a71fdd96609a78ee9e75e232581e1d85f3776 Mon Sep 17 00:00:00 2001 From: hoeckxer Date: Tue, 12 Jan 2021 08:24:11 +0100 Subject: [PATCH 124/183] Reformat code on new version --- freqtrade/strategy/interface.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 40debe78f..2f1326f48 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -489,8 +489,7 @@ class IStrategy(ABC): def ignore_expired_candle(self, latest_date: datetime, current_time: datetime, timeframe_seconds: int, buy: bool): - if self.ignore_buying_expired_candle_after \ - and buy: + if self.ignore_buying_expired_candle_after and buy: time_delta = current_time - (latest_date + timedelta(seconds=timeframe_seconds)) return time_delta.total_seconds() > self.ignore_buying_expired_candle_after else: From 60ea32e39811ea8c48bcab90fa201373731ba423 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 Jan 2021 19:05:25 +0100 Subject: [PATCH 125/183] Improve wording --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5127ba5e0..25f1e7c46 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -679,7 +679,7 @@ freqtrade When working with larger timeframes (for example 1h or more) and using a low `max_open_trades` value, the last candle can be processed as soon as a trade slot becomes available. When processing the last candle, this can lead to a situation where it may not be desirable to use the buy signal on that candle. For example, when using a condition in your strategy where you use a cross-over, that point may have passed too long ago for you to start a trade on it. -In these situations, you can enable the functionality to ignore candles that are beyond a specified period by setting `ask_strategy.ignore_buying_expired_candle_after` to a positive number, indicating the number of seconds after which the candle becomes expired. +In these situations, you can enable the functionality to ignore candles that are beyond a specified period by setting `ask_strategy.ignore_buying_expired_candle_after` to a positive number, indicating the number of seconds after which the buy signal becomes expired. For example, if your strategy is using a 1h timeframe, and you only want to buy within the first 5 minutes when a new candle comes in, you can add the following configuration to your strategy: From ac43591c44bb0d90f21b1dbaa7c046bc2da3b9b4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 Jan 2021 19:24:37 +0100 Subject: [PATCH 126/183] Fix failing api when max_open_trades is unlimited --- freqtrade/rpc/rpc.py | 3 ++- tests/rpc/test_rpc_apiserver.py | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 19c90fff0..7c9be89aa 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -649,7 +649,8 @@ class RPC: trades = Trade.get_open_trades() return { 'current': len(trades), - 'max': float(self._freqtrade.config['max_open_trades']), + 'max': (int(self._freqtrade.config['max_open_trades']) + if self._freqtrade.config['max_open_trades'] != float('inf') else -1), 'total_stake': sum((trade.open_rate * trade.amount) for trade in trades) } diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 2212d4a79..4eb9a6fc8 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -364,14 +364,19 @@ def test_api_count(botclient, mocker, ticker, fee, markets): assert_response(rc) assert rc.json()["current"] == 0 - assert rc.json()["max"] == 1.0 + assert rc.json()["max"] == 1 # Create some test data ftbot.enter_positions() rc = client_get(client, f"{BASE_URI}/count") assert_response(rc) - assert rc.json()["current"] == 1.0 - assert rc.json()["max"] == 1.0 + assert rc.json()["current"] == 1 + assert rc.json()["max"] == 1 + + ftbot.config['max_open_trades'] = float('inf') + rc = client_get(client, f"{BASE_URI}/count") + assert rc.json()["max"] == -1 + def test_api_locks(botclient): From 47a06c6213e9b885551dd9b35711dcf5fb77b90e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 Jan 2021 19:27:49 +0100 Subject: [PATCH 127/183] Fix enable/reenable of swagger UI endpoint --- docs/rest-api.md | 2 +- freqtrade/rpc/api_server/webserver.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index a013bf358..2c7142c61 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -266,7 +266,7 @@ whitelist ## OpenAPI interface -To enable the builtin openAPI interface, specify `"enable_openapi": true` in the api_server configuration. +To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. This will enable the Swagger UI at the `/docs` endpoint. By default, that's running at http://localhost:8080/docs/ - but it'll depend on your settings. ## Advanced API usage using JWT tokens diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 97dfa444d..9c0779274 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -30,8 +30,7 @@ class ApiServer(RPCHandler): api_config = self._config['api_server'] self.app = FastAPI(title="Freqtrade API", - openapi_url='openapi.json' if api_config.get( - 'enable_openapi', False) else None, + docs_url='/docs' if api_config.get('enable_openapi', False) else None, redoc_url=None, ) self.configure_app(self.app, self._config) From adb3fb123e0c5720a20c6ae3a2791d65592d22f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 Jan 2021 19:35:02 +0100 Subject: [PATCH 128/183] Fix typo --- tests/rpc/test_rpc_apiserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 4eb9a6fc8..5460519fe 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -378,7 +378,6 @@ def test_api_count(botclient, mocker, ticker, fee, markets): assert rc.json()["max"] == -1 - def test_api_locks(botclient): ftbot, client = botclient From 950c5c0113655128f06bbfc12929fd5ccb324bb8 Mon Sep 17 00:00:00 2001 From: tejeshreddy Date: Wed, 13 Jan 2021 16:09:03 +0530 Subject: [PATCH 129/183] fix: edge doc typos --- docs/edge.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index fd6d2cf7d..6f01fcf65 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -1,6 +1,6 @@ # Edge positioning -The `Edge Positioning` module uses probability to calculate your win rate and risk reward ration. It will use these statistics to control your strategy trade entry points, position side and, stoploss. +The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss. !!! Warning `Edge positioning` is not compatible with dynamic (volume-based) whitelist. @@ -55,7 +55,7 @@ Similarly, we can discover the set of losing trades $T_{lose}$ as follows: $$ T_{lose} = \{o \in O | o \leq 0\} $$ !!! Example - In a section where a strategy made three transactions $O = \{3.5, -1, 15, 0\}$:
+ In a section where a strategy made four transactions $O = \{3.5, -1, 15, 0\}$:
$T_{win} = \{3.5, 15\}$
$T_{lose} = \{-1, 0\}$
@@ -206,7 +206,7 @@ Let's say the stake currency is **ETH** and there is $10$ **ETH** on the wallet. - The strategy detects a sell signal in the **XLM/ETH** market. The bot exits **Trade 1** for a profit of $1$ **ETH**. The total capital in the wallet becomes $11$ **ETH** and the available capital for trading becomes $5.5$ **ETH**. -- **Trade 4** The strategy detects a new buy signal int the **XLM/ETH** market. `Edge Positioning` calculates the stoploss of $2%$, and the position size of $0.055 / 0.02 = 2.75$ **ETH**. +- **Trade 4** The strategy detects a new buy signal int the **XLM/ETH** market. `Edge Positioning` calculates the stoploss of $2\%$, and the position size of $0.055 / 0.02 = 2.75$ **ETH**. ## Configurations From f3de0dd3eb8d55ef085258073d6506bd887f8cc8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 14 Jan 2021 06:53:40 +0100 Subject: [PATCH 130/183] Fix support for protections in hyperopt closes #4208 --- freqtrade/optimize/backtesting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a689786ec..6913b2f4b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -180,6 +180,7 @@ class Backtesting: Backtesting setup method - called once for every call to "backtest()". """ PairLocks.use_db = False + PairLocks.timeframe = self.config['timeframe'] Trade.use_db = False if enable_protections: # Reset persisted data - used for protections only From 6d1fba140949740a129a75cae41d4adbed8a9745 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 18 Sep 2020 07:43:42 +0200 Subject: [PATCH 131/183] Remove unnecessary log output tests --- tests/optimize/test_backtesting.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 376390664..1f8f7cfd8 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -353,8 +353,6 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: backtesting.start() # check the logs, that will contain the backtest result exists = [ - 'Using stake_currency: BTC ...', - 'Using stake_amount: 0.001 ...', 'Backtesting with data from 2017-11-14 21:17:00 ' 'up to 2017-11-14 22:59:00 (0 days)..' ] @@ -722,8 +720,6 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', - 'Using stake_currency: BTC ...', - 'Using stake_amount: 0.001 ...', 'Loading data from 2017-11-14 20:57:00 ' 'up to 2017-11-14 22:58:00 (0 days)..', 'Backtesting with data from 2017-11-14 21:17:00 ' @@ -786,8 +782,6 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', - 'Using stake_currency: BTC ...', - 'Using stake_amount: 0.001 ...', 'Loading data from 2017-11-14 20:57:00 ' 'up to 2017-11-14 22:58:00 (0 days)..', 'Backtesting with data from 2017-11-14 21:17:00 ' @@ -865,8 +859,6 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', - 'Using stake_currency: BTC ...', - 'Using stake_amount: 0.001 ...', 'Loading data from 2017-11-14 20:57:00 ' 'up to 2017-11-14 22:58:00 (0 days)..', 'Backtesting with data from 2017-11-14 21:17:00 ' From 9d4cdcad10df21c149df59bdd7c4adde46553459 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 18 Sep 2020 07:44:11 +0200 Subject: [PATCH 132/183] Extract backtesting of one strategy --- freqtrade/optimize/backtesting.py | 93 ++++++++++++++++--------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6913b2f4b..fff3914a5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -76,6 +76,8 @@ class Backtesting: # Reset keys for backtesting remove_credentials(self.config) self.strategylist: List[IStrategy] = [] + self.all_results: Dict[str, Dict] = {} + self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) dataprovider = DataProvider(self.config, self.exchange) @@ -424,6 +426,47 @@ class Backtesting: return DataFrame.from_records(trades, columns=BacktestResult._fields) + def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange): + logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) + self._set_strategy(strat) + + # Use max_open_trades in backtesting, except --disable-max-market-positions is set + if self.config.get('use_max_market_positions', True): + # Must come from strategy config, as the strategy may modify this setting. + max_open_trades = self.strategy.config['max_open_trades'] + else: + logger.info( + 'Ignoring max_open_trades (--disable-max-market-positions was used) ...') + max_open_trades = 0 + + # need to reprocess data every time to populate signals + preprocessed = self.strategy.ohlcvdata_to_dataframe(data) + + # Trim startup period from analyzed dataframe + for pair, df in preprocessed.items(): + preprocessed[pair] = trim_dataframe(df, timerange) + min_date, max_date = history.get_timerange(preprocessed) + + logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'({(max_date - min_date).days} days)..') + # Execute backtest and store results + results = self.backtest( + processed=preprocessed, + stake_amount=self.config['stake_amount'], + start_date=min_date.datetime, + end_date=max_date.datetime, + max_open_trades=max_open_trades, + position_stacking=self.config.get('position_stacking', False), + enable_protections=self.config.get('enable_protections', False), + ) + self.all_results[self.strategy.get_strategy_name()] = { + 'results': results, + 'config': self.strategy.config, + 'locks': PairLocks.locks, + } + return min_date, max_date + def start(self) -> None: """ Run backtesting end-to-end @@ -431,55 +474,15 @@ class Backtesting: """ data: Dict[str, Any] = {} - logger.info('Using stake_currency: %s ...', self.config['stake_currency']) - logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - - position_stacking = self.config.get('position_stacking', False) - data, timerange = self.load_bt_data() - all_results = {} + min_date = None + max_date = None for strat in self.strategylist: - logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) - self._set_strategy(strat) + min_date, max_date = self.backtest_one_strategy(strat, data, timerange) - # Use max_open_trades in backtesting, except --disable-max-market-positions is set - if self.config.get('use_max_market_positions', True): - # Must come from strategy config, as the strategy may modify this setting. - max_open_trades = self.strategy.config['max_open_trades'] - else: - logger.info( - 'Ignoring max_open_trades (--disable-max-market-positions was used) ...') - max_open_trades = 0 - - # need to reprocess data every time to populate signals - preprocessed = self.strategy.ohlcvdata_to_dataframe(data) - - # Trim startup period from analyzed dataframe - for pair, df in preprocessed.items(): - preprocessed[pair] = trim_dataframe(df, timerange) - min_date, max_date = history.get_timerange(preprocessed) - - logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'({(max_date - min_date).days} days)..') - # Execute backtest and print results - results = self.backtest( - processed=preprocessed, - stake_amount=self.config['stake_amount'], - start_date=min_date.datetime, - end_date=max_date.datetime, - max_open_trades=max_open_trades, - position_stacking=position_stacking, - enable_protections=self.config.get('enable_protections', False), - ) - all_results[self.strategy.get_strategy_name()] = { - 'results': results, - 'config': self.strategy.config, - 'locks': PairLocks.locks, - } - - stats = generate_backtest_stats(data, all_results, min_date=min_date, max_date=max_date) + stats = generate_backtest_stats(data, self.all_results, + min_date=min_date, max_date=max_date) if self.config.get('export', False): store_backtest_stats(self.config['exportfilename'], stats) From baa1142afa75c577ee4bfd50f283a0e345253ebd Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 18 Sep 2020 07:45:47 +0200 Subject: [PATCH 133/183] Use preprocessed to get min/max date in hyperopt --- freqtrade/optimize/backtesting.py | 4 ++++ freqtrade/optimize/hyperopt.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fff3914a5..87eb8cb05 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -152,6 +152,10 @@ class Backtesting: self.strategy.order_types['stoploss_on_exchange'] = False def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]: + """ + Loads backtest data and returns the data combined with the timerange + as tuple. + """ timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2a2f5b472..d4b9f4c3b 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -650,7 +650,7 @@ class Hyperopt: # Trim startup period from analyzed dataframe for pair, df in preprocessed.items(): preprocessed[pair] = trim_dataframe(df, timerange) - min_date, max_date = get_timerange(data) + min_date, max_date = get_timerange(preprocessed) logger.info(f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' From 914710625940d8ac95a886c6f43c2f001f69af76 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Oct 2020 19:50:37 +0200 Subject: [PATCH 134/183] call bot_loop_start() in backtesting to allow setup-code to run --- docs/bot-basics.md | 5 +++-- freqtrade/optimize/backtesting.py | 3 +++ tests/optimize/test_backtesting.py | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/bot-basics.md b/docs/bot-basics.md index 44f493456..86fb18645 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -49,8 +49,9 @@ This loop will be repeated again and again until the bot is stopped. [backtesting](backtesting.md) or [hyperopt](hyperopt.md) do only part of the above logic, since most of the trading operations are fully simulated. * Load historic data for configured pairlist. -* Calculate indicators (calls `populate_indicators()`). -* Calls `populate_buy_trend()` and `populate_sell_trend()` +* Calls `bot_loop_start()` once. +* Calculate indicators (calls `populate_indicators()` once per pair). +* Calculate buy / sell signals (calls `populate_buy_trend()` and `populate_sell_trend()` once per pair) * Loops per candle simulating entry and exit points. * Generate backtest report output diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 87eb8cb05..cb021c9ff 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -26,6 +26,7 @@ from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType +from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper logger = logging.getLogger(__name__) @@ -434,6 +435,8 @@ class Backtesting: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) self._set_strategy(strat) + strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)() + # Use max_open_trades in backtesting, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): # Must come from strategy config, as the strategy may modify this setting. diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 1f8f7cfd8..e55e166d9 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -350,6 +350,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: default_conf['timerange'] = '-1510694220' backtesting = Backtesting(default_conf) + backtesting.strategy.bot_loop_start = MagicMock() backtesting.start() # check the logs, that will contain the backtest result exists = [ @@ -359,6 +360,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: for line in exists: assert log_has(line, caplog) assert backtesting.strategy.dp._pairlists is not None + assert backtesting.strategy.bot_loop_start.call_count == 1 def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None: From 0b65fe6afe4b4d219dbc0d58277f1c2ad545ce2d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 13 Jan 2021 07:47:03 +0100 Subject: [PATCH 135/183] Capture backtest start / end time --- freqtrade/optimize/backtesting.py | 6 +++++- freqtrade/optimize/optimize_reports.py | 6 ++++++ tests/optimize/test_optimize_reports.py | 5 ++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cb021c9ff..106d0f200 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,7 +6,7 @@ This module contains the backtesting logic import logging from collections import defaultdict from copy import deepcopy -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, NamedTuple, Optional, Tuple from pandas import DataFrame @@ -433,6 +433,7 @@ class Backtesting: def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange): logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) + backtest_start_time = datetime.now(timezone.utc) self._set_strategy(strat) strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)() @@ -467,10 +468,13 @@ class Backtesting: position_stacking=self.config.get('position_stacking', False), enable_protections=self.config.get('enable_protections', False), ) + backtest_end_time = datetime.now(timezone.utc) self.all_results[self.strategy.get_strategy_name()] = { 'results': results, 'config': self.strategy.config, 'locks': PairLocks.locks, + 'backtest_start_time': int(backtest_start_time.timestamp()), + 'backtest_end_time': int(backtest_end_time.timestamp()), } return min_date, max_date diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 6c70b7c84..a2bb6277e 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -282,6 +282,9 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], 'backtest_end_ts': max_date.int_timestamp * 1000, 'backtest_days': backtest_days, + 'backtest_run_start_ts': content['backtest_start_time'], + 'backtest_run_end_ts': content['backtest_end_time'], + 'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0, 'market_change': market_change, 'pairlist': list(btdata.keys()), @@ -290,6 +293,9 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], 'max_open_trades': (config['max_open_trades'] if config['max_open_trades'] != float('inf') else -1), 'timeframe': config['timeframe'], + 'timerange': config.get('timerange', ''), + 'enable_protections': config.get('enable_protections', False), + 'strategy_name': strategy, # Parameters relevant for backtesting 'stoploss': config['stoploss'], 'trailing_stop': config.get('trailing_stop', False), diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index e194e7de4..f184cb125 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -77,7 +77,10 @@ def test_generate_backtest_stats(default_conf, testdatadir): SellType.ROI, SellType.FORCE_SELL] }), 'config': default_conf, - 'locks': []} + 'locks': [], + 'backtest_start_time': Arrow.utcnow().int_timestamp, + 'backtest_end_time': Arrow.utcnow().int_timestamp, + } } timerange = TimeRange.parse_timerange('1510688220-1510700340') min_date = Arrow.fromtimestamp(1510688220) From f72d53351cd9c10530db8e5f613685d9009abbcf Mon Sep 17 00:00:00 2001 From: nas- Date: Fri, 15 Jan 2021 00:13:11 +0100 Subject: [PATCH 136/183] Added ability to keep invalid pairs while expanding expand_pairlist --- docs/configuration.md | 2 +- docs/includes/pairlists.md | 2 +- freqtrade/exchange/exchange.py | 2 +- freqtrade/plugins/pairlist/IPairList.py | 10 +-- freqtrade/plugins/pairlist/StaticPairList.py | 4 +- .../plugins/pairlist/pairlist_helpers.py | 37 ++++++++--- freqtrade/plugins/pairlistmanager.py | 33 +++++++--- tests/exchange/test_exchange.py | 63 +++++++++---------- tests/plugins/test_pairlist.py | 32 ++++++++++ 9 files changed, 128 insertions(+), 57 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 13c7eb47b..33d117c91 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -81,7 +81,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String -| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used by VolumePairList (see [below](#pairlists-and-pairlist-handlers)).
**Datatype:** List +| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting.Supports regex pairs as */BTC. Not used by VolumePairList (see [below](#pairlists-and-pairlist-handlers)).
**Datatype:** List | `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#pairlists-and-pairlist-handlers)).
**Datatype:** List | `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict | `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 8919c4e3d..1ad38cc73 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -35,7 +35,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged #### Static Pair List -By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. +By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. Also the pairlist does support wildcards (in regex-style) - so `*/BTC` will include all pairs with BTC as a stake. It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`. diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 489f70528..436c8e4e9 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -336,7 +336,7 @@ class Exchange: if not self.markets: logger.warning('Unable to validate pairs (assuming they are correct).') return - extended_pairs = expand_pairlist(pairs, list(self.markets)) + extended_pairs = expand_pairlist(pairs, list(self.markets), keep_invalid=True) invalid_pairs = [] for pair in extended_pairs: # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index f9809aeb1..95d776ae6 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -124,19 +124,21 @@ class IPairList(LoggingMixin, ABC): """ return self._pairlistmanager.verify_blacklist(pairlist, logmethod) - def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]: + def verify_whitelist(self, pairlist: List[str], logmethod, + keep_invalid: bool = False) -> List[str]: """ Proxy method to verify_whitelist for easy access for child classes. :param pairlist: Pairlist to validate - :param logmethod: Function that'll be called, `logger.info` or `logger.warning`. + :param logmethod: Function that'll be called, `logger.info` or `logger.warning` + :param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes. :return: pairlist - whitelisted pairs """ - return self._pairlistmanager.verify_whitelist(pairlist, logmethod) + return self._pairlistmanager.verify_whitelist(pairlist, logmethod, keep_invalid) def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary - :param whitelist: the sorted list of pairs the user might want to trade + :param pairlist: the sorted list of pairs the user might want to trade :return: the list of pairs the user wants to trade without those unavailable or black_listed """ diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index c216a322d..c5ced48c9 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -50,7 +50,9 @@ class StaticPairList(IPairList): :return: List of pairs """ if self._allow_inactive: - return self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info) + return self.verify_whitelist( + self._config['exchange']['pair_whitelist'], logger.info, keep_invalid=True + ) else: return self._whitelist_for_active_markets( self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info)) diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py index 3352777f0..04320fe16 100644 --- a/freqtrade/plugins/pairlist/pairlist_helpers.py +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -2,22 +2,41 @@ import re from typing import List -def expand_pairlist(wildcardpl: List[str], available_pairs: List[str]) -> List[str]: +def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], + keep_invalid: bool = False) -> List[str]: """ Expand pairlist potentially containing wildcards based on available markets. This will implicitly filter all pairs in the wildcard-list which are not in available_pairs. :param wildcardpl: List of Pairlists, which may contain regex :param available_pairs: List of all available pairs (`exchange.get_markets().keys()`) + :param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes :return expanded pairlist, with Regexes from wildcardpl applied to match all available pairs. :raises: ValueError if a wildcard is invalid (like '*/BTC' - which should be `.*/BTC`) """ result = [] - for pair_wc in wildcardpl: - try: - comp = re.compile(pair_wc) - result += [ - pair for pair in available_pairs if re.match(comp, pair) - ] - except re.error as err: - raise ValueError(f"Wildcard error in {pair_wc}, {err}") + if keep_invalid: + for pair_wc in wildcardpl: + try: + comp = re.compile(pair_wc) + result_partial = [ + pair for pair in available_pairs if re.match(comp, pair) + ] + # Add all matching pairs. + # If there are no matching pairs (Pair not on exchange) keep it. + result += result_partial or [pair_wc] + except re.error as err: + raise ValueError(f"Wildcard error in {pair_wc}, {err}") + + for element in result: + if not re.fullmatch(r'^[A-Za-z0-9/-]+$', element): + result.remove(element) + else: + for pair_wc in wildcardpl: + try: + comp = re.compile(pair_wc) + result += [ + pair for pair in available_pairs if re.match(comp, pair) + ] + except re.error as err: + raise ValueError(f"Wildcard error in {pair_wc}, {err}") return result diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index c471d7c51..15e186e6c 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -59,9 +59,15 @@ class PairListManager(): """The expanded blacklist (including wildcard expansion)""" return expand_pairlist(self._blacklist, self._exchange.get_markets().keys()) + @property + def expanded_whitelist_keep_invalid(self) -> List[str]: + """The expanded whitelist (including wildcard expansion), maintaining invalid pairs""" + return expand_pairlist(self._whitelist, self._exchange.get_markets().keys(), + keep_invalid=True) + @property def expanded_whitelist(self) -> List[str]: - """The expanded whitelist (including wildcard expansion)""" + """The expanded whitelist (including wildcard expansion), filtering invalid pairs""" return expand_pairlist(self._whitelist, self._exchange.get_markets().keys()) @property @@ -134,19 +140,30 @@ class PairListManager(): pairlist.remove(pair) return pairlist - def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]: + def verify_whitelist(self, pairlist: List[str], logmethod, + keep_invalid: bool = False) -> List[str]: """ Verify and remove items from pairlist - returning a filtered pairlist. Logs a warning or info depending on `aswarning`. Pairlist Handlers explicitly using this method shall use `logmethod=logger.info` to avoid spamming with warning messages - :return: pairlist - blacklisted pairs + :param pairlist: Pairlist to validate + :param logmethod: Function that'll be called, `logger.info` or `logger.warning` + :param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes. + :return: pairlist - whitelisted pairs """ - try: - whitelist = self.expanded_whitelist - except ValueError as err: - logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") - return [] + if keep_invalid: + try: + whitelist = self.expanded_whitelist_keep_invalid + except ValueError as err: + logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") + return [] + else: + try: + whitelist = self.expanded_whitelist + except ValueError as err: + logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") + return [] return whitelist def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes: diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8cd2a9bca..9d655997f 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -505,38 +505,37 @@ def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs d Exchange(default_conf) -# This cannot happen anymore as expand_pairlist implicitly filters out unavaliablie pairs -# def test_validate_pairs_not_available(default_conf, mocker): -# api_mock = MagicMock() -# type(api_mock).markets = PropertyMock(return_value={ -# 'XRP/BTC': {'inactive': True, 'base': 'XRP', 'quote': 'BTC'} -# }) -# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) -# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') -# mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') -# mocker.patch('freqtrade.exchange.Exchange._load_async_markets') -# -# with pytest.raises(OperationalException, match=r'not available'): -# Exchange(default_conf) -# -# -# def test_validate_pairs_exception(default_conf, mocker, caplog): -# caplog.set_level(logging.INFO) -# api_mock = MagicMock() -# mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) -# -# type(api_mock).markets = PropertyMock(return_value={}) -# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) -# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') -# mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') -# mocker.patch('freqtrade.exchange.Exchange._load_async_markets') -# -# with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): -# Exchange(default_conf) -# -# mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})) -# Exchange(default_conf) -# assert log_has('Unable to validate pairs (assuming they are correct).', caplog) +def test_validate_pairs_not_available(default_conf, mocker): + api_mock = MagicMock() + type(api_mock).markets = PropertyMock(return_value={ + 'XRP/BTC': {'inactive': True, 'base': 'XRP', 'quote': 'BTC'} + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + + with pytest.raises(OperationalException, match=r'not available'): + Exchange(default_conf) + + +def test_validate_pairs_exception(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + api_mock = MagicMock() + mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) + + type(api_mock).markets = PropertyMock(return_value={}) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + + with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): + Exchange(default_conf) + + mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={})) + Exchange(default_conf) + assert log_has('Unable to validate pairs (assuming they are correct).', caplog) def test_validate_pairs_restricted(default_conf, mocker, caplog): diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index d822f8319..e20e42c60 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -853,3 +853,35 @@ def test_expand_pairlist(wildcardlist, pairs, expected): expand_pairlist(wildcardlist, pairs) else: assert sorted(expand_pairlist(wildcardlist, pairs)) == sorted(expected) + + +@pytest.mark.parametrize('wildcardlist,pairs,expected', [ + (['BTC/USDT'], + ['BTC/USDT'], + ['BTC/USDT']), + (['BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETH/USDT']), + (['BTC/USDT', 'ETH/USDT'], + ['BTC/USDT'], ['BTC/USDT', 'ETH/USDT']), # Test one too many + (['.*/USDT'], + ['BTC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETH/USDT']), # Wildcard simple + (['.*C/USDT'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETC/USDT']), # Wildcard exclude one + (['.*UP/USDT', 'BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'], + ['BTC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT']), # Wildcard exclude one + (['BTC/.*', 'ETH/.*'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP'], + ['BTC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP']), # Wildcard exclude one + (['*UP/USDT', 'BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'], + None), + (['HELLO/WORLD'], [], ['HELLO/WORLD']) # Invalid pair kept +]) +def test_expand_pairlist_keep_invalid(wildcardlist, pairs, expected): + if expected is None: + with pytest.raises(ValueError, match=r'Wildcard error in \*UP/USDT,'): + expand_pairlist(wildcardlist, pairs, keep_invalid=True) + else: + assert sorted(expand_pairlist(wildcardlist, pairs, keep_invalid=True)) == sorted(expected) From bf5868c96da6b0194c6bafa319f09bddef00d350 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 15 Jan 2021 06:56:15 +0100 Subject: [PATCH 137/183] Add testcase for nonexisting pairs on whitelist --- docs/configuration.md | 2 +- docs/includes/pairlists.md | 2 +- freqtrade/plugins/pairlistmanager.py | 16 ++++++---------- tests/plugins/test_pairlist.py | 26 +++++++++++++++++++++++++- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 33d117c91..e655182b8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -81,7 +81,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String -| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting.Supports regex pairs as */BTC. Not used by VolumePairList (see [below](#pairlists-and-pairlist-handlers)).
**Datatype:** List +| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Supports regex pairs as `.*/BTC`. Not used by VolumePairList (see [below](#pairlists-and-pairlist-handlers)).
**Datatype:** List | `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#pairlists-and-pairlist-handlers)).
**Datatype:** List | `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict | `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 1ad38cc73..2653406e7 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -35,7 +35,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged #### Static Pair List -By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. Also the pairlist does support wildcards (in regex-style) - so `*/BTC` will include all pairs with BTC as a stake. +By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. The pairlist also supports wildcards (in regex-style) - so `.*/BTC` will include all pairs with BTC as a stake. It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`. diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 15e186e6c..7ce77da59 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -152,18 +152,14 @@ class PairListManager(): :param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes. :return: pairlist - whitelisted pairs """ - if keep_invalid: - try: + try: + if keep_invalid: whitelist = self.expanded_whitelist_keep_invalid - except ValueError as err: - logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") - return [] - else: - try: + else: whitelist = self.expanded_whitelist - except ValueError as err: - logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") - return [] + except ValueError as err: + logger.error(f"Pair whitelist contains an invalid Wildcard: {err}") + return [] return whitelist def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes: diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index e20e42c60..910a9580c 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -156,6 +156,31 @@ def test_refresh_static_pairlist(mocker, markets, static_pl_conf): assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist +@pytest.mark.parametrize('pairs,expected', [ + (['NOEXIST/BTC', r'\+WHAT/BTC'], + ['ETH/BTC', 'TKN/BTC', 'TRST/BTC', 'NOEXIST/BTC', 'SWT/BTC', 'BCC/BTC', 'HOT/BTC']), + (['NOEXIST/BTC', r'*/BTC'], # This is an invalid regex + []), +]) +def test_refresh_static_pairlist_noexist(mocker, markets, static_pl_conf, pairs, expected, caplog): + + static_pl_conf['pairlists'][0]['allow_inactive'] = True + static_pl_conf['exchange']['pair_whitelist'] += pairs + freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + markets=PropertyMock(return_value=markets), + ) + freqtrade.pairlists.refresh_pairlist() + + # Ensure all except those in whitelist are removed + assert set(expected) == set(freqtrade.pairlists.whitelist) + assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist + if not expected: + assert log_has_re(r'Pair whitelist contains an invalid Wildcard: Wildcard error.*', caplog) + + def test_invalid_blacklist(mocker, markets, static_pl_conf, caplog): static_pl_conf['exchange']['pair_blacklist'] = ['*/BTC'] freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) @@ -165,7 +190,6 @@ def test_invalid_blacklist(mocker, markets, static_pl_conf, caplog): markets=PropertyMock(return_value=markets), ) freqtrade.pairlists.refresh_pairlist() - # List ordered by BaseVolume whitelist = [] # Ensure all except those in whitelist are removed assert set(whitelist) == set(freqtrade.pairlists.whitelist) From d74376726a090b2489ddf61d45c10838d69b0fc1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 15 Jan 2021 20:47:12 +0100 Subject: [PATCH 138/183] api-server should fully support max_open_trades=-1 --- freqtrade/rpc/rpc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 7c9be89aa..1e304f01b 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -121,7 +121,8 @@ class RPC: 'dry_run': config['dry_run'], 'stake_currency': config['stake_currency'], 'stake_amount': config['stake_amount'], - 'max_open_trades': config['max_open_trades'], + 'max_open_trades': (config['max_open_trades'] + if config['max_open_trades'] != float('inf') else -1), 'minimal_roi': config['minimal_roi'].copy() if 'minimal_roi' in config else {}, 'stoploss': config.get('stoploss'), 'trailing_stop': config.get('trailing_stop'), From 9f338ba6ed79b65ac15e067fa316ae665a3ac139 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jan 2021 10:01:31 +0100 Subject: [PATCH 139/183] Debug random test failure in CI --- tests/rpc/test_rpc_apiserver.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 5460519fe..518eb189e 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -523,13 +523,17 @@ def test_api_logs(botclient): assert isinstance(rc.json()['logs'][0][3], str) assert isinstance(rc.json()['logs'][0][4], str) - rc = client_get(client, f"{BASE_URI}/logs?limit=5") - assert_response(rc) - assert len(rc.json()) == 2 - assert 'logs' in rc.json() + rc1 = client_get(client, f"{BASE_URI}/logs?limit=5") + assert_response(rc1) + assert len(rc1.json()) == 2 + assert 'logs' in rc1.json() # Using a fixed comparison here would make this test fail! - assert rc.json()['log_count'] == 5 - assert len(rc.json()['logs']) == rc.json()['log_count'] + if rc1.json()['log_count'] == 0: + # Help debugging random test failure + print(f"{rc.json()=}") + print(f"{rc1.json()=}") + assert rc1.json()['log_count'] == 5 + assert len(rc1.json()['logs']) == rc1.json()['log_count'] def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): From 572f5f91861c0d2a5558b75b91c5853828803a38 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jan 2021 10:05:47 +0100 Subject: [PATCH 140/183] Fix fstring syntax error --- tests/rpc/test_rpc_apiserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 518eb189e..c782f5431 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -530,8 +530,8 @@ def test_api_logs(botclient): # Using a fixed comparison here would make this test fail! if rc1.json()['log_count'] == 0: # Help debugging random test failure - print(f"{rc.json()=}") - print(f"{rc1.json()=}") + print(f"rc={rc.json()}") + print(f"rc1={rc1.json()}") assert rc1.json()['log_count'] == 5 assert len(rc1.json()['logs']) == rc1.json()['log_count'] From 53c208197d12560f2e4b39338831ac509f3960bf Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 16 Jan 2021 16:19:49 +0100 Subject: [PATCH 141/183] Add bot_name setting allows naming the bot to simply differentiate when running different bots. --- config.json.example | 1 + config_binance.json.example | 1 + config_full.json.example | 1 + config_kraken.json.example | 1 + docs/configuration.md | 1 + freqtrade/constants.py | 1 + freqtrade/rpc/api_server/api_schemas.py | 1 + freqtrade/rpc/rpc.py | 1 + freqtrade/templates/base_config.json.j2 | 1 + tests/rpc/test_rpc_apiserver.py | 1 + 10 files changed, 10 insertions(+) diff --git a/config.json.example b/config.json.example index fc59a4d5b..0f0bbec4b 100644 --- a/config.json.example +++ b/config.json.example @@ -85,6 +85,7 @@ "username": "freqtrader", "password": "SuperSecurePassword" }, + "bot_name": "freqtrade", "initial_state": "running", "forcebuy_enable": false, "internals": { diff --git a/config_binance.json.example b/config_binance.json.example index 954634def..83c9748d7 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -90,6 +90,7 @@ "username": "freqtrader", "password": "SuperSecurePassword" }, + "bot_name": "freqtrade", "initial_state": "running", "forcebuy_enable": false, "internals": { diff --git a/config_full.json.example b/config_full.json.example index ef791d267..6593750b4 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -177,6 +177,7 @@ "username": "freqtrader", "password": "SuperSecurePassword" }, + "bot_name": "freqtrade", "db_url": "sqlite:///tradesv3.sqlite", "initial_state": "running", "forcebuy_enable": false, diff --git a/config_kraken.json.example b/config_kraken.json.example index 4b33eb592..3cd90e5d3 100644 --- a/config_kraken.json.example +++ b/config_kraken.json.example @@ -95,6 +95,7 @@ "username": "freqtrader", "password": "SuperSecurePassword" }, + "bot_name": "freqtrade", "initial_state": "running", "forcebuy_enable": false, "internals": { diff --git a/docs/configuration.md b/docs/configuration.md index 6a05fc3a3..9a3126618 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -110,6 +110,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `api_server.verbosity` | Logging verbosity. `info` will print all RPC Calls, while "error" will only display errors.
**Datatype:** Enum, either `info` or `error`. Defaults to `info`. | `api_server.username` | Username for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `api_server.password` | Password for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `bot_name` | Name of the bot. Passed via API to a client - can be shown to distinguish / name bots.
*Defaults to `freqtrade`*
**Datatype:** String | `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
**Datatype:** String, SQLAlchemy connect string | `initial_state` | Defines the initial application state. More information below.
*Defaults to `stopped`.*
**Datatype:** Enum, either `stopped` or `running` | `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below.
**Datatype:** Boolean diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d48ab635e..69301ca0e 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -116,6 +116,7 @@ CONF_SCHEMA = { 'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1}, 'trailing_stop_positive_offset': {'type': 'number', 'minimum': 0, 'maximum': 1}, 'trailing_only_offset_is_reached': {'type': 'boolean'}, + 'bot_name': {'type': 'string'}, 'unfilledtimeout': { 'type': 'object', 'properties': { diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 45f160008..c9e8aaceb 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -131,6 +131,7 @@ class ShowConfig(BaseModel): forcebuy_enabled: bool ask_strategy: Dict[str, Any] bid_strategy: Dict[str, Any] + bot_name: str state: str runmode: str diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 1e304f01b..bee04ddb6 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -129,6 +129,7 @@ class RPC: 'trailing_stop_positive': config.get('trailing_stop_positive'), 'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset'), 'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached'), + 'bot_name': config.get('bot_name', 'freqtrade'), 'timeframe': config.get('timeframe'), 'timeframe_ms': timeframe_to_msecs(config['timeframe'] ) if 'timeframe' in config else '', diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index b362690f9..f920843b2 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -63,6 +63,7 @@ "username": "", "password": "" }, + "bot_name": "freqtrade", "initial_state": "running", "forcebuy_enable": false, "internals": { diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index c782f5431..935f43885 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -414,6 +414,7 @@ def test_api_show_config(botclient, mocker): assert rc.json()['timeframe_ms'] == 300000 assert rc.json()['timeframe_min'] == 5 assert rc.json()['state'] == 'running' + assert rc.json()['bot_name'] == 'freqtrade' assert not rc.json()['trailing_stop'] assert 'bid_strategy' in rc.json() assert 'ask_strategy' in rc.json() From 389db2fe7de254d3aa09dbf9c8035697469561a0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 17 Jan 2021 19:11:00 +0100 Subject: [PATCH 142/183] Enhance wording of docker quickstart --- docs/docker_quickstart.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index e25e1b050..85f5a4a2d 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -8,9 +8,7 @@ Start by downloading and installing Docker CE for your platform: * [Windows](https://docs.docker.com/docker-for-windows/install/) * [Linux](https://docs.docker.com/install/) -Optionally, [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the [docker quick start guide](#docker-quick-start). - -Once you have Docker installed, simply prepare the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below. +To simplify running freqtrade, please install [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the below [docker quick start guide](#docker-quick-start). ## Freqtrade with docker-compose @@ -83,7 +81,8 @@ The `SampleStrategy` is run by default. !!! Warning "`SampleStrategy` is just a demo!" The `SampleStrategy` is there for your reference and give you ideas for your own strategy. - Please always backtest the strategy and use dry-run for some time before risking real money! + Please always backtest your strategy and use dry-run for some time before risking real money! + You will find more information about Strategy development in the [Strategy documentation](strategy-customization.md). Once this is done, you're ready to launch the bot in trading mode (Dry-run or Live-trading, depending on your answer to the corresponding question you made above). @@ -93,16 +92,16 @@ docker-compose up -d #### Docker-compose logs -Logs will be located at: `user_data/logs/freqtrade.log`. -You can check the latest log with the command `docker-compose logs -f`. +Logs will be written to: `user_data/logs/freqtrade.log`. +You can also check the latest log with the command `docker-compose logs -f`. #### Database -The database will be at: `user_data/tradesv3.sqlite` +The database will be located at: `user_data/tradesv3.sqlite` #### Updating freqtrade with docker-compose -To update freqtrade when using `docker-compose` is as simple as running the following 2 commands: +Updating freqtrade when using `docker-compose` is as simple as running the following 2 commands: ``` bash # Download the latest image @@ -120,7 +119,7 @@ This will first pull the latest image, and will then restart the container with Advanced users may edit the docker-compose file further to include all possible options or arguments. -All possible freqtrade arguments will be available by running `docker-compose run --rm freqtrade `. +All freqtrade arguments will be available by running `docker-compose run --rm freqtrade `. !!! Note "`docker-compose run --rm`" Including `--rm` will remove the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command). From a8bae3a38161d3fc6396c0819bb24df45b505af1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 17 Jan 2021 20:31:27 +0100 Subject: [PATCH 143/183] Don't update trade fees for dry-run orders --- freqtrade/freqtradebot.py | 4 ++++ tests/test_freqtradebot.py | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6dc8eacf9..926f22225 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -268,6 +268,10 @@ class FreqtradeBot(LoggingMixin): Update closed trades without close fees assigned. Only acts when Orders are in the database, otherwise the last orderid is unknown. """ + if self.config['dry_run']: + # Updating open orders in dry-run does not make sense and will fail. + return + trades: List[Trade] = Trade.get_sold_trades_without_assigned_fees() for trade in trades: diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e6aff3352..6257a7e0b 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -4368,6 +4368,19 @@ def test_update_closed_trades_without_assigned_fees(mocker, default_conf, fee): freqtrade.update_closed_trades_without_assigned_fees() + # Does nothing for dry-run + trades = Trade.get_trades().all() + assert len(trades) == MOCK_TRADE_COUNT + for trade in trades: + assert trade.fee_open_cost is None + assert trade.fee_open_currency is None + assert trade.fee_close_cost is None + assert trade.fee_close_currency is None + + freqtrade.config['dry_run'] = False + + freqtrade.update_closed_trades_without_assigned_fees() + trades = Trade.get_trades().all() assert len(trades) == MOCK_TRADE_COUNT From 6d40814dbf3059fe8c23bede6a98d433ecd15651 Mon Sep 17 00:00:00 2001 From: Andreas Brunner Date: Sun, 17 Jan 2021 20:39:35 +0100 Subject: [PATCH 144/183] extend status bot command to query specific trades --- freqtrade/rpc/rpc.py | 10 +++++++--- freqtrade/rpc/telegram.py | 8 +++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 1e304f01b..03108b0f4 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -144,13 +144,17 @@ class RPC: } return val - def _rpc_trade_status(self) -> List[Dict[str, Any]]: + def _rpc_trade_status(self, trade_ids=None) -> List[Dict[str, Any]]: """ Below follows the RPC backend it is prefixed with rpc_ to raise awareness that it is a remotely exposed function """ - # Fetch open trade - trades = Trade.get_open_trades() + # Fetch open trades + if trade_ids: + trades = Trade.get_trades(trade_filter=Trade.id.in_(trade_ids)) + else: + trades = Trade.get_open_trades() + if not trades: raise RPCException('no active trade') else: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 7ec67e5d0..d304188c2 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -277,7 +277,13 @@ class Telegram(RPCHandler): return try: - results = self._rpc._rpc_trade_status() + + # Check if there's at least one numerical ID provided. If so, try to get only these trades. + trade_ids = None + if context.args and len(context.args) > 0: + trade_ids = [i for i in context.args if i.isnumeric()] + + results = self._rpc._rpc_trade_status(trade_ids=trade_ids) messages = [] for r in results: From 3ea33d1737ee82750d4d6b4d1707f6cc49d27cf1 Mon Sep 17 00:00:00 2001 From: Andreas Brunner Date: Sun, 17 Jan 2021 21:15:17 +0100 Subject: [PATCH 145/183] updating doc and help with new /status argument --- README.md | 2 +- docs/telegram-usage.md | 1 + freqtrade/rpc/telegram.py | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1031e4d67..c61116219 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Telegram is not mandatory. However, this is a great way to control your bot. Mor - `/start`: Starts the trader. - `/stop`: Stops the trader. - `/stopbuy`: Stop entering new trades. -- `/status [table]`: Lists all open trades. +- `/status |[table]`: Lists all or specific open trades. - `/profit`: Lists cumulative profit from all finished trades - `/forcesell |all`: Instantly sells the given trade (Ignoring `minimum_roi`). - `/performance`: Show performance of each finished trade grouped by pair diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 40481684d..57f2e98bd 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -137,6 +137,7 @@ official commands. You can ask at any moment for help with `/help`. | `/show_config` | Shows part of the current configuration with relevant settings to operation | `/logs [limit]` | Show last log messages. | `/status` | Lists all open trades +| `/status ` | Lists one or more specific trade. Separate multiple with a blank space. | `/status table` | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) Pending sell orders are marked with a double asterisk (**) | `/trades [limit]` | List all recently closed trades in a table format. | `/delete ` | Delete a specific trade from the Database. Tries to close open orders. Requires manual handling of this trade on the exchange. diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index d304188c2..abdaa948d 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -821,7 +821,9 @@ class Telegram(RPCHandler): "Optionally takes a rate at which to buy.` \n") message = ("*/start:* `Starts the trader`\n" "*/stop:* `Stops the trader`\n" - "*/status [table]:* `Lists all open trades`\n" + "*/status |[table]:* `Lists all open trades`\n" + " * :* `Lists one or more specific trades.`\n" + " `Separate multiple with a blank space.`\n" " *table :* `will display trades in a table`\n" " `pending buy orders are marked with an asterisk (*)`\n" " `pending sell orders are marked with a double asterisk (**)`\n" From d21eff0d5231ca49c4a7a7544b6bc17e1137dbbb Mon Sep 17 00:00:00 2001 From: Andreas Brunner Date: Sun, 17 Jan 2021 21:21:31 +0100 Subject: [PATCH 146/183] fix, if an non existing trade_id is provided --- freqtrade/rpc/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 03108b0f4..69e3d057d 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -151,7 +151,7 @@ class RPC: """ # Fetch open trades if trade_ids: - trades = Trade.get_trades(trade_filter=Trade.id.in_(trade_ids)) + trades = Trade.get_trades(trade_filter=Trade.id.in_(trade_ids)).all() else: trades = Trade.get_open_trades() From eb95d970e9b029cb7b9778be1732f96e677cacfe Mon Sep 17 00:00:00 2001 From: Andreas Brunner Date: Sun, 17 Jan 2021 21:26:55 +0100 Subject: [PATCH 147/183] flake8 beautify --- freqtrade/rpc/telegram.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index abdaa948d..c310f9803 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -278,7 +278,8 @@ class Telegram(RPCHandler): try: - # Check if there's at least one numerical ID provided. If so, try to get only these trades. + # Check if there's at least one numerical ID provided. + # If so, try to get only these trades. trade_ids = None if context.args and len(context.args) > 0: trade_ids = [i for i in context.args if i.isnumeric()] From 296a6bd43cb8b5f01303225fdb576b16d66c5cb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jan 2021 05:37:29 +0000 Subject: [PATCH 148/183] Bump coveralls from 2.2.0 to 3.0.0 Bumps [coveralls](https://github.com/coveralls-clients/coveralls-python) from 2.2.0 to 3.0.0. - [Release notes](https://github.com/coveralls-clients/coveralls-python/releases) - [Changelog](https://github.com/TheKevJames/coveralls-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/coveralls-clients/coveralls-python/compare/2.2.0...3.0.0) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 883c3089f..749450289 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,7 @@ -r requirements-plot.txt -r requirements-hyperopt.txt -coveralls==2.2.0 +coveralls==3.0.0 flake8==3.8.4 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.2.1 From 6a8e495102fc0e79ab127727097eb3886cf19acc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jan 2021 05:37:37 +0000 Subject: [PATCH 149/183] Bump plotly from 4.14.1 to 4.14.3 Bumps [plotly](https://github.com/plotly/plotly.py) from 4.14.1 to 4.14.3. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v4.14.1...v4.14.3) Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index 3e31a24ae..6693a593d 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==4.14.1 +plotly==4.14.3 From 7f8dbce367fe564acc13ace1cae07361c89f4139 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jan 2021 05:37:45 +0000 Subject: [PATCH 150/183] Bump mkdocs-material from 6.2.4 to 6.2.5 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 6.2.4 to 6.2.5. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/6.2.4...6.2.5) Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index adf4bc96c..6fef05f0c 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,3 +1,3 @@ -mkdocs-material==6.2.4 +mkdocs-material==6.2.5 mdx_truly_sane_lists==1.2 pymdown-extensions==8.1 From 8b5f8937cc4d643b5f45f81d7b0f85cf76a4e7b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jan 2021 05:37:48 +0000 Subject: [PATCH 151/183] Bump pyjwt from 2.0.0 to 2.0.1 Bumps [pyjwt](https://github.com/jpadilla/pyjwt) from 2.0.0 to 2.0.1. - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.0.0...2.0.1) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 74295c68e..5a44639a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ sdnotify==0.3.2 # API Server fastapi==0.63.0 uvicorn==0.13.3 -pyjwt==2.0.0 +pyjwt==2.0.1 # Support for colorized terminal output colorama==0.4.4 From 994b4013adcc83e8ad3a99c76a6974822e9bdcea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jan 2021 05:38:06 +0000 Subject: [PATCH 152/183] Bump ccxt from 1.40.30 to 1.40.74 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.40.30 to 1.40.74. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst) - [Commits](https://github.com/ccxt/ccxt/compare/1.40.30...1.40.74) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 74295c68e..3c91868a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.19.5 pandas==1.2.0 -ccxt==1.40.30 +ccxt==1.40.74 aiohttp==3.7.3 SQLAlchemy==1.3.22 python-telegram-bot==13.1 From 10104927c914f44792933a4e513bf2886b72e1b7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 18 Jan 2021 07:46:19 +0000 Subject: [PATCH 153/183] Fix devcontainer closes #4230 --- .devcontainer/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b333dc19d..19e09c969 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -3,13 +3,15 @@ FROM freqtradeorg/freqtrade:develop # Install dependencies COPY requirements-dev.txt /freqtrade/ RUN apt-get update \ - && apt-get -y install git sudo vim \ + && apt-get -y install git mercurial sudo vim \ && apt-get clean \ && pip install autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir \ && useradd -u 1000 -U -m ftuser \ && mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \ && echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \ && echo "export HISTFILE=~/commandhistory/.bash_history" >> /home/ftuser/.bashrc \ + && mv /root/.local /home/ftuser/.local/ \ + && chown ftuser:ftuser -R /home/ftuser/.local/ \ && chown ftuser: -R /home/ftuser/ USER ftuser From a68a546dd9fca6b1cf4abb67f4a4036280251bf8 Mon Sep 17 00:00:00 2001 From: Andreas Brunner Date: Mon, 18 Jan 2021 15:26:53 +0100 Subject: [PATCH 154/183] _rpc_trade_status argument datatype optimizations --- freqtrade/rpc/rpc.py | 2 +- freqtrade/rpc/telegram.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 69e3d057d..f74d63408 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -144,7 +144,7 @@ class RPC: } return val - def _rpc_trade_status(self, trade_ids=None) -> List[Dict[str, Any]]: + def _rpc_trade_status(self, trade_ids: List[int] = []) -> List[Dict[str, Any]]: """ Below follows the RPC backend it is prefixed with rpc_ to raise awareness that it is a remotely exposed function diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c310f9803..99f9a8a91 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -280,9 +280,9 @@ class Telegram(RPCHandler): # Check if there's at least one numerical ID provided. # If so, try to get only these trades. - trade_ids = None + trade_ids = [] if context.args and len(context.args) > 0: - trade_ids = [i for i in context.args if i.isnumeric()] + trade_ids = [int(i) for i in context.args if i.isnumeric()] results = self._rpc._rpc_trade_status(trade_ids=trade_ids) From cd8d4da46696d64033e9d3a963d61948b2ee3cd3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 19 Jan 2021 19:44:16 +0100 Subject: [PATCH 155/183] Add test for /status functionality --- tests/rpc/test_rpc_telegram.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index a21a19e3a..1c34b6b26 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -205,13 +205,14 @@ def test_telegram_status(default_conf, update, mocker) -> None: assert msg_mock.call_count == 1 context = MagicMock() - # /status table 2 3 - context.args = ["table", "2", "3"] + # /status table + context.args = ["table"] telegram._status(update=update, context=context) assert status_table.call_count == 1 def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: + default_conf['max_open_trades'] = 3 mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, @@ -252,8 +253,23 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: assert 'Close Rate' not in ''.join(lines) assert 'Close Profit' not in ''.join(lines) - assert msg_mock.call_count == 1 + assert msg_mock.call_count == 3 assert 'ETH/BTC' in msg_mock.call_args_list[0][0][0] + assert 'LTC/BTC' in msg_mock.call_args_list[1][0][0] + + msg_mock.reset_mock() + context = MagicMock() + context.args = ["2", "3"] + + telegram._status(update=update, context=context) + + lines = msg_mock.call_args_list[0][0][0].split('\n') + assert '' not in lines + assert 'Close Rate' not in ''.join(lines) + assert 'Close Profit' not in ''.join(lines) + + assert msg_mock.call_count == 2 + assert 'LTC/BTC' in msg_mock.call_args_list[0][0][0] def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: From 7c99e6f0e6d8ffb4b2f256894a049a5e578aa259 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 19 Jan 2021 20:49:28 +0100 Subject: [PATCH 156/183] Avoid random test failure --- tests/plugins/test_pairlocks.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/plugins/test_pairlocks.py b/tests/plugins/test_pairlocks.py index bd103b21e..dfcbff0ed 100644 --- a/tests/plugins/test_pairlocks.py +++ b/tests/plugins/test_pairlocks.py @@ -11,11 +11,10 @@ from freqtrade.persistence.models import PairLock @pytest.mark.usefixtures("init_persistence") def test_PairLocks(use_db): PairLocks.timeframe = '5m' + PairLocks.use_db = use_db # No lock should be present if use_db: assert len(PairLock.query.all()) == 0 - else: - PairLocks.use_db = False assert PairLocks.use_db == use_db @@ -88,10 +87,9 @@ def test_PairLocks(use_db): def test_PairLocks_getlongestlock(use_db): PairLocks.timeframe = '5m' # No lock should be present + PairLocks.use_db = use_db if use_db: assert len(PairLock.query.all()) == 0 - else: - PairLocks.use_db = False assert PairLocks.use_db == use_db From 86b3306a3b7910360a4abeb05eb1763a6f08f924 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 19 Jan 2021 22:07:10 +0100 Subject: [PATCH 157/183] Small doc refactoring --- docs/configuration.md | 58 +++++++++++---------------------------- docs/strategy-advanced.md | 26 ++++++++++++++++++ 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9a3126618..781435271 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -277,6 +277,22 @@ before asking the strategy if we should buy or a sell an asset. After each wait every opened trade wether or not we should sell, and for all the remaining pairs (either the dynamic list of pairs or the static list of pairs) if we should buy. +### Ignoring expired candles + +When working with larger timeframes (for example 1h or more) and using a low `max_open_trades` value, the last candle can be processed as soon as a trade slot becomes available. When processing the last candle, this can lead to a situation where it may not be desirable to use the buy signal on that candle. For example, when using a condition in your strategy where you use a cross-over, that point may have passed too long ago for you to start a trade on it. + +In these situations, you can enable the functionality to ignore candles that are beyond a specified period by setting `ask_strategy.ignore_buying_expired_candle_after` to a positive number, indicating the number of seconds after which the buy signal becomes expired. + +For example, if your strategy is using a 1h timeframe, and you only want to buy within the first 5 minutes when a new candle comes in, you can add the following configuration to your strategy: + +``` json + "ask_strategy":{ + "ignore_buying_expired_candle_after": 300, + "price_side": "bid", + // ... + }, +``` + ### Understand order_types The `order_types` configuration parameter maps actions (`buy`, `sell`, `stoploss`, `emergencysell`) to order-types (`market`, `limit`, ...) as well as configures stoploss to be on the exchange and defines stoploss on exchange update interval in seconds. @@ -676,48 +692,6 @@ export HTTPS_PROXY="http://addr:port" freqtrade ``` -## Ignoring expired candles - -When working with larger timeframes (for example 1h or more) and using a low `max_open_trades` value, the last candle can be processed as soon as a trade slot becomes available. When processing the last candle, this can lead to a situation where it may not be desirable to use the buy signal on that candle. For example, when using a condition in your strategy where you use a cross-over, that point may have passed too long ago for you to start a trade on it. - -In these situations, you can enable the functionality to ignore candles that are beyond a specified period by setting `ask_strategy.ignore_buying_expired_candle_after` to a positive number, indicating the number of seconds after which the buy signal becomes expired. - -For example, if your strategy is using a 1h timeframe, and you only want to buy within the first 5 minutes when a new candle comes in, you can add the following configuration to your strategy: - -``` jsonc - "ask_strategy":{ - "ignore_buying_expired_candle_after" = 300 # 5 minutes - "price_side": "bid", - // ... - }, -``` - -## Embedding Strategies - -Freqtrade provides you with with an easy way to embed the strategy into your configuration file. -This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, -in your chosen config file. - -### Encoding a string as BASE64 - -This is a quick example, how to generate the BASE64 string in python - -```python -from base64 import urlsafe_b64encode - -with open(file, 'r') as f: - content = f.read() -content = urlsafe_b64encode(content.encode('utf-8')) -``` - -The variable 'content', will contain the strategy file in a BASE64 encoded form. Which can now be set in your configurations file as following - -```json -"strategy": "NameOfStrategy:BASE64String" -``` - -Please ensure that 'NameOfStrategy' is identical to the strategy name! - ## Next step Now you have configured your config.json, the next step is to [start your bot](bot-usage.md). diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 2431274d7..25d217d34 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -398,3 +398,29 @@ class MyAwesomeStrategy2(MyAwesomeStrategy): ``` Both attributes and methods may be overridden, altering behavior of the original strategy in a way you need. + +## Embedding Strategies + +Freqtrade provides you with with an easy way to embed the strategy into your configuration file. +This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, +in your chosen config file. + +### Encoding a string as BASE64 + +This is a quick example, how to generate the BASE64 string in python + +```python +from base64 import urlsafe_b64encode + +with open(file, 'r') as f: + content = f.read() +content = urlsafe_b64encode(content.encode('utf-8')) +``` + +The variable 'content', will contain the strategy file in a BASE64 encoded form. Which can now be set in your configurations file as following + +```json +"strategy": "NameOfStrategy:BASE64String" +``` + +Please ensure that 'NameOfStrategy' is identical to the strategy name! From 7c80eeea950a95138639c34f64f57d2e5fb14c18 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 19 Jan 2021 22:51:12 +0100 Subject: [PATCH 158/183] Add use_custom_stoploss to optimize_report --- freqtrade/optimize/optimize_reports.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index a2bb6277e..96ddb91a0 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -302,6 +302,7 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], 'trailing_stop_positive': config.get('trailing_stop_positive'), 'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0), 'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False), + 'use_custom_stoploss': config.get('use_custom_stoploss', False), 'minimal_roi': config['minimal_roi'], 'use_sell_signal': config['ask_strategy']['use_sell_signal'], 'sell_profit_only': config['ask_strategy']['sell_profit_only'], From 992d6b801806fde03d8a4f011ebb91190def45ca Mon Sep 17 00:00:00 2001 From: Tijmen van den Brink Date: Wed, 20 Jan 2021 09:24:30 +0100 Subject: [PATCH 159/183] Small improvement to MaxDrawDown protection --- docs/includes/protections.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 95fa53ad2..3ae456c42 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -65,7 +65,7 @@ The below example stops trading for all pairs for 4 candles after the last trade `MaxDrawdown` uses all trades within `lookback_period` in minutes (or in candles when using `lookback_period_candles`) to determine the maximum drawdown. If the drawdown is below `max_allowed_drawdown`, trading will stop for `stop_duration` in minutes (or in candles when using `stop_duration_candles`) after the last trade - assuming that the bot needs some time to let markets recover. -The below sample stops trading for 12 candles if max-drawdown is > 20% considering all trades within the last 48 candles. If desired, `lookback_period` and/or `stop_duration` can be used. +The below sample stops trading for 12 candles if max-drawdown is > 20% considering all trades - with a minimum of `trade_limit` trades - within the last 48 candles. If desired, `lookback_period` and/or `stop_duration` can be used. ```json "protections": [ @@ -77,7 +77,6 @@ The below sample stops trading for 12 candles if max-drawdown is > 20% consideri "max_allowed_drawdown": 0.2 }, ], - ``` #### Low Profit Pairs From 5f5f75e147360a57dffe911c3a4bdb0de39680e1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 20 Jan 2021 13:57:53 +0100 Subject: [PATCH 160/183] Improve wording in protections documentation --- docs/includes/protections.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 3ae456c42..de34383ac 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -65,7 +65,7 @@ The below example stops trading for all pairs for 4 candles after the last trade `MaxDrawdown` uses all trades within `lookback_period` in minutes (or in candles when using `lookback_period_candles`) to determine the maximum drawdown. If the drawdown is below `max_allowed_drawdown`, trading will stop for `stop_duration` in minutes (or in candles when using `stop_duration_candles`) after the last trade - assuming that the bot needs some time to let markets recover. -The below sample stops trading for 12 candles if max-drawdown is > 20% considering all trades - with a minimum of `trade_limit` trades - within the last 48 candles. If desired, `lookback_period` and/or `stop_duration` can be used. +The below sample stops trading for 12 candles if max-drawdown is > 20% considering all pairs - with a minimum of `trade_limit` trades - within the last 48 candles. If desired, `lookback_period` and/or `stop_duration` can be used. ```json "protections": [ From 5c0f98b518274fc4f46bcee0449cffb17cc21d00 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 20 Jan 2021 19:30:43 +0100 Subject: [PATCH 161/183] Blacklist Poloniex - as ccxt does not provide a fetch_order endpoint --- freqtrade/exchange/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index ce0fde9e4..c66db860f 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -21,6 +21,7 @@ BAD_EXCHANGES = { "hitbtc": "This API cannot be used with Freqtrade. " "Use `hitbtc2` exchange id to access this exchange.", "phemex": "Does not provide history. ", + "poloniex": "Does not provide fetch_order endpoint to fetch both open and closed orders.", **dict.fromkeys([ 'adara', 'anxpro', From fd379d36ac59a2d38e51486ad1a3a2dceab50458 Mon Sep 17 00:00:00 2001 From: Davide Date: Thu, 21 Jan 2021 12:27:22 +0100 Subject: [PATCH 162/183] Fixed quickstart link in docs --- docs/docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docker.md b/docs/docker.md index f4699cf4c..1298296f1 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -1,7 +1,7 @@ ## Freqtrade with docker without docker-compose !!! Warning - The below documentation is provided for completeness and assumes that you are familiar with running docker containers. If you're just starting out with Docker, we recommend to follow the [Quickstart](docker.md) instructions. + The below documentation is provided for completeness and assumes that you are familiar with running docker containers. If you're just starting out with Docker, we recommend to follow the [Quickstart](docker_quickstart.md) instructions. ### Download the official Freqtrade docker image From c42241986e9f4d9b03e18b87c7cb87cbfd95f7db Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 21 Jan 2021 19:20:38 +0100 Subject: [PATCH 163/183] further investigate random test failure --- tests/rpc/test_rpc_apiserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 935f43885..f5b9a58f3 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -529,7 +529,7 @@ def test_api_logs(botclient): assert len(rc1.json()) == 2 assert 'logs' in rc1.json() # Using a fixed comparison here would make this test fail! - if rc1.json()['log_count'] == 0: + if rc1.json()['log_count'] < 5: # Help debugging random test failure print(f"rc={rc.json()}") print(f"rc1={rc1.json()}") From e94e2dd383e787c254464cb5a43ca514895da269 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 22 Jan 2021 17:32:57 +0100 Subject: [PATCH 164/183] Remove docker config without compose --- docs/docker.md | 201 ----------------------------------- docs/docker_quickstart.md | 7 +- docs/installation.md | 4 +- docs/windows_installation.md | 4 +- mkdocs.yml | 1 - 5 files changed, 10 insertions(+), 207 deletions(-) delete mode 100644 docs/docker.md diff --git a/docs/docker.md b/docs/docker.md deleted file mode 100644 index 1298296f1..000000000 --- a/docs/docker.md +++ /dev/null @@ -1,201 +0,0 @@ -## Freqtrade with docker without docker-compose - -!!! Warning - The below documentation is provided for completeness and assumes that you are familiar with running docker containers. If you're just starting out with Docker, we recommend to follow the [Quickstart](docker_quickstart.md) instructions. - -### Download the official Freqtrade docker image - -Pull the image from docker hub. - -Branches / tags available can be checked out on [Dockerhub tags page](https://hub.docker.com/r/freqtradeorg/freqtrade/tags/). - -```bash -docker pull freqtradeorg/freqtrade:stable -# Optionally tag the repository so the run-commands remain shorter -docker tag freqtradeorg/freqtrade:stable freqtrade -``` - -To update the image, simply run the above commands again and restart your running container. - -Should you require additional libraries, please [build the image yourself](#build-your-own-docker-image). - -!!! Note "Docker image update frequency" - The official docker images with tags `stable`, `develop` and `latest` are automatically rebuild once a week to keep the base image up-to-date. - In addition to that, every merge to `develop` will trigger a rebuild for `develop` and `latest`. - -### Prepare the configuration files - -Even though you will use docker, you'll still need some files from the github repository. - -#### Clone the git repository - -Linux/Mac/Windows with WSL - -```bash -git clone https://github.com/freqtrade/freqtrade.git -``` - -Windows with docker - -```bash -git clone --config core.autocrlf=input https://github.com/freqtrade/freqtrade.git -``` - -#### Copy `config.json.example` to `config.json` - -```bash -cd freqtrade -cp -n config.json.example config.json -``` - -> To understand the configuration options, please refer to the [Bot Configuration](configuration.md) page. - -#### Create your database file - -=== "Dry-Run" - ``` bash - touch tradesv3.dryrun.sqlite - ``` - -=== "Production" - ``` bash - touch tradesv3.sqlite - ``` - - -!!! Warning "Database File Path" - Make sure to use the path to the correct database file when starting the bot in Docker. - -### Build your own Docker image - -Best start by pulling the official docker image from dockerhub as explained [here](#download-the-official-docker-image) to speed up building. - -To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image. - -```bash -docker build -t freqtrade -f docker/Dockerfile.technical . -``` - -If you are developing using Docker, use `docker/Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies: - -```bash -docker build -f docker/Dockerfile.develop -t freqtrade-dev . -``` - -!!! Warning "Include your config file manually" - For security reasons, your configuration file will not be included in the image, you will need to bind mount it. It is also advised to bind mount an SQLite database file (see [5. Run a restartable docker image](#run-a-restartable-docker-image)") to keep it between updates. - -#### Verify the Docker image - -After the build process you can verify that the image was created with: - -```bash -docker images -``` - -The output should contain the freqtrade image. - -### Run the Docker image - -You can run a one-off container that is immediately deleted upon exiting with the following command (`config.json` must be in the current working directory): - -```bash -docker run --rm -v `pwd`/config.json:/freqtrade/config.json -it freqtrade -``` - -!!! Warning - In this example, the database will be created inside the docker instance and will be lost when you refresh your image. - -#### Adjust timezone - -By default, the container will use UTC timezone. -If you would like to change the timezone use the following commands: - -=== "Linux" - ``` bash - -v /etc/timezone:/etc/timezone:ro - - # Complete command: - docker run --rm -v /etc/timezone:/etc/timezone:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade - ``` - -=== "MacOS" - ```bash - docker run --rm -e TZ=`ls -la /etc/localtime | cut -d/ -f8-9` -v `pwd`/config.json:/freqtrade/config.json -it freqtrade - ``` - -!!! Note "MacOS Issues" - The OSX Docker versions after 17.09.1 have a known issue whereby `/etc/localtime` cannot be shared causing Docker to not start.
- A work-around for this is to start with the MacOS command above - More information on this docker issue and work-around can be read [here](https://github.com/docker/for-mac/issues/2396). - -### Run a restartable docker image - -To run a restartable instance in the background (feel free to place your configuration and database files wherever it feels comfortable on your filesystem). - -#### 1. Move your config file and database - -The following will assume that you place your configuration / database files to `~/.freqtrade`, which is a hidden directory in your home directory. Feel free to use a different directory and replace the directory in the upcomming commands. - -```bash -mkdir ~/.freqtrade -mv config.json ~/.freqtrade -mv tradesv3.sqlite ~/.freqtrade -``` - -#### 2. Run the docker image - -```bash -docker run -d \ - --name freqtrade \ - -v ~/.freqtrade/config.json:/freqtrade/config.json \ - -v ~/.freqtrade/user_data/:/freqtrade/user_data \ - -v ~/.freqtrade/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \ - freqtrade trade --db-url sqlite:///tradesv3.sqlite --strategy MyAwesomeStrategy -``` - -!!! Note - When using docker, it's best to specify `--db-url` explicitly to ensure that the database URL and the mounted database file match. - -!!! Note - All available bot command line parameters can be added to the end of the `docker run` command. - -!!! Note - You can define a [restart policy](https://docs.docker.com/config/containers/start-containers-automatically/) in docker. It can be useful in some cases to use the `--restart unless-stopped` flag (crash of freqtrade or reboot of your system). - -### Monitor your Docker instance - -You can use the following commands to monitor and manage your container: - -```bash -docker logs freqtrade -docker logs -f freqtrade -docker restart freqtrade -docker stop freqtrade -docker start freqtrade -``` - -For more information on how to operate Docker, please refer to the [official Docker documentation](https://docs.docker.com/). - -!!! Note - You do not need to rebuild the image for configuration changes, it will suffice to edit `config.json` and restart the container. - -### Backtest with docker - -The following assumes that the download/setup of the docker image have been completed successfully. -Also, backtest-data should be available at `~/.freqtrade/user_data/`. - -```bash -docker run -d \ - --name freqtrade \ - -v /etc/localtime:/etc/localtime:ro \ - -v ~/.freqtrade/config.json:/freqtrade/config.json \ - -v ~/.freqtrade/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \ - -v ~/.freqtrade/user_data/:/freqtrade/user_data/ \ - freqtrade backtesting --strategy AwsomelyProfitableStrategy -``` - -Head over to the [Backtesting Documentation](backtesting.md) for more details. - -!!! Note - Additional bot command line parameters can be appended after the image name (`freqtrade` in the above example). diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 85f5a4a2d..9cccfa93d 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -69,7 +69,7 @@ The last 2 steps in the snippet create the directory with `user_data`, as well a !!! Question "How to edit the bot configuration?" You can edit the configuration at any time, which is available as `user_data/config.json` (within the directory `ft_userdata`) when using the above configuration. - You can also change the both Strategy and commands by editing the `docker-compose.yml` file. + You can also change the both Strategy and commands by editing the command section of your `docker-compose.yml` file. #### Adding a custom strategy @@ -90,6 +90,11 @@ Once this is done, you're ready to launch the bot in trading mode (Dry-run or Li docker-compose up -d ``` +#### Monitoring the bot + +You can check for running instances with `docker-compose ps`. +This should list the service `freqtrade` as `running`. If that's not the case, best check the logs (see next point). + #### Docker-compose logs Logs will be written to: `user_data/logs/freqtrade.log`. diff --git a/docs/installation.md b/docs/installation.md index a23399441..8cb6724cb 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,7 +2,7 @@ This page explains how to prepare your environment for running the bot. -Please consider using the prebuilt [docker images](docker.md) to get started quickly while trying out freqtrade evaluating how it operates. +Please consider using the prebuilt [docker images](docker_quickstart.md) to get started quickly while evaluating how freqtrade works. ## Prerequisite @@ -210,7 +210,7 @@ If this is the first time you run the bot, ensure you are running it in Dry-run freqtrade trade -c config.json ``` -*Note*: If you run the bot on a server, you should consider using [Docker](docker.md) or a terminal multiplexer like `screen` or [`tmux`](https://en.wikipedia.org/wiki/Tmux) to avoid that the bot is stopped on logout. +*Note*: If you run the bot on a server, you should consider using [Docker compose](docker_quickstart.md) or a terminal multiplexer like `screen` or [`tmux`](https://en.wikipedia.org/wiki/Tmux) to avoid that the bot is stopped on logout. #### 7. (Optional) Post-installation Tasks diff --git a/docs/windows_installation.md b/docs/windows_installation.md index 5341ce96b..168938973 100644 --- a/docs/windows_installation.md +++ b/docs/windows_installation.md @@ -1,4 +1,4 @@ -We **strongly** recommend that Windows users use [Docker](docker.md) as this will work much easier and smoother (also more secure). +We **strongly** recommend that Windows users use [Docker](docker_quickstart.md) as this will work much easier and smoother (also more secure). If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work. Otherwise, try the instructions below. @@ -52,6 +52,6 @@ error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Unfortunately, many packages requiring compilation don't provide a pre-built wheel. It is therefore mandatory to have a C/C++ compiler installed and available for your python environment to use. -The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building C code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or [docker](docker.md) first. +The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building C code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or [docker compose](docker_quickstart.md) first. --- diff --git a/mkdocs.yml b/mkdocs.yml index 96cfa7651..4545e8d84 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,7 +3,6 @@ nav: - Home: index.md - Quickstart with Docker: docker_quickstart.md - Installation: - - Docker without docker-compose: docker.md - Linux/MacOS/Raspberry: installation.md - Windows: windows_installation.md - Freqtrade Basics: bot-basics.md From bec9b580b042a6baff158b651b0605632e2147d2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 22 Jan 2021 17:34:41 +0100 Subject: [PATCH 165/183] sell_profit_offset should be documented in the strategy override section --- docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 781435271..660dd6171 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -16,8 +16,7 @@ In some advanced use cases, multiple configuration files can be specified and us If you used the [Quick start](installation.md/#quick-start) method for installing the bot, the installation script should have already created the default configuration file (`config.json`) for you. -If default configuration file is not created we recommend you to copy and use the `config.json.example` as a template -for your bot configuration. +If default configuration file is not created we recommend you to use `freqtrade new-config --config config.json` to generate a basic configuration file. The Freqtrade configuration file is to be written in the JSON format. @@ -147,6 +146,7 @@ Values set in the configuration file always overwrite values set in the strategy * `protections` * `use_sell_signal` (ask_strategy) * `sell_profit_only` (ask_strategy) +* `sell_profit_offset` (ask_strategy) * `ignore_roi_if_buy_signal` (ask_strategy) * `ignore_buying_expired_candle_after` (ask_strategy) From 371b374ea610a059f80d1b20cc591cb72737ba1a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 22 Jan 2021 19:12:34 +0100 Subject: [PATCH 166/183] Remove unused config setup from setup.sh --- setup.sh | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) diff --git a/setup.sh b/setup.sh index b270146c1..d0ca1f643 100755 --- a/setup.sh +++ b/setup.sh @@ -202,52 +202,6 @@ function test_and_fix_python_on_mac() { fi } -function config_generator() { - - echo "Starting to generate config.json" - echo - echo "Generating General configuration" - echo "-------------------------" - default_max_trades=3 - read -p "Max open trades: (Default: $default_max_trades) " max_trades - max_trades=${max_trades:-$default_max_trades} - - default_stake_amount=0.05 - read -p "Stake amount: (Default: $default_stake_amount) " stake_amount - stake_amount=${stake_amount:-$default_stake_amount} - - default_stake_currency="BTC" - read -p "Stake currency: (Default: $default_stake_currency) " stake_currency - stake_currency=${stake_currency:-$default_stake_currency} - - default_fiat_currency="USD" - read -p "Fiat currency: (Default: $default_fiat_currency) " fiat_currency - fiat_currency=${fiat_currency:-$default_fiat_currency} - - echo - echo "Generating exchange config " - echo "------------------------" - read -p "Exchange API key: " api_key - read -p "Exchange API Secret: " api_secret - - echo - echo "Generating Telegram config" - echo "-------------------------" - read -p "Telegram Token: " token - read -p "Telegram Chat_id: " chat_id - - sed -e "s/\"max_open_trades\": 3,/\"max_open_trades\": $max_trades,/g" \ - -e "s/\"stake_amount\": 0.05,/\"stake_amount\": $stake_amount,/g" \ - -e "s/\"stake_currency\": \"BTC\",/\"stake_currency\": \"$stake_currency\",/g" \ - -e "s/\"fiat_display_currency\": \"USD\",/\"fiat_display_currency\": \"$fiat_currency\",/g" \ - -e "s/\"your_exchange_key\"/\"$api_key\"/g" \ - -e "s/\"your_exchange_secret\"/\"$api_secret\"/g" \ - -e "s/\"your_telegram_token\"/\"$token\"/g" \ - -e "s/\"your_telegram_chat_id\"/\"$chat_id\"/g" \ - -e "s/\"dry_run\": false,/\"dry_run\": true,/g" config.json.example > config.json - -} - function config() { echo "-------------------------" From 31e0b09643796733a8d4918222347a90856c25b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 22 Jan 2021 19:18:21 +0100 Subject: [PATCH 167/183] Rename config.json.example it's really the config dedicated to bittrex, so the name should reflect this in beeing config_bittrex.json.example --- .github/workflows/ci.yml | 12 ++--- .travis.yml | 4 +- MANIFEST.in | 1 - build_helpers/publish_docker.sh | 2 +- ...son.example => config_bittrex.json.example | 0 freqtrade/configuration/config_validation.py | 2 +- tests/commands/test_commands.py | 54 +++++++++---------- tests/test_arguments.py | 2 +- tests/test_main.py | 20 +++---- tests/test_plotting.py | 4 +- 10 files changed, 50 insertions(+), 51 deletions(-) rename config.json.example => config_bittrex.json.example (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 961dfef71..3f294347a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,13 +79,13 @@ jobs: - name: Backtesting run: | - cp config.json.example config.json + cp config_bittrex.json.example config.json freqtrade create-userdir --userdir user_data freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy - name: Hyperopt run: | - cp config.json.example config.json + cp config_bittrex.json.example config.json freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all @@ -171,13 +171,13 @@ jobs: - name: Backtesting run: | - cp config.json.example config.json + cp config_bittrex.json.example config.json freqtrade create-userdir --userdir user_data freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy - name: Hyperopt run: | - cp config.json.example config.json + cp config_bittrex.json.example config.json freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all @@ -238,13 +238,13 @@ jobs: - name: Backtesting run: | - cp config.json.example config.json + cp config_bittrex.json.example config.json freqtrade create-userdir --userdir user_data freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy - name: Hyperopt run: | - cp config.json.example config.json + cp config_bittrex.json.example config.json freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all diff --git a/.travis.yml b/.travis.yml index 94239e33f..03a8df49b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,12 +26,12 @@ jobs: # - coveralls || true name: pytest - script: - - cp config.json.example config.json + - cp config_bittrex.json.example config.json - freqtrade create-userdir --userdir user_data - freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy name: backtest - script: - - cp config.json.example config.json + - cp config_bittrex.json.example config.json - freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily name: hyperopt diff --git a/MANIFEST.in b/MANIFEST.in index c67f5258f..2f59bcc7a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ include LICENSE include README.md -include config.json.example recursive-include freqtrade *.py recursive-include freqtrade/templates/ *.j2 *.ipynb diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index ac0cd2461..9bc1aa0a6 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -30,7 +30,7 @@ if [ $? -ne 0 ]; then fi # Run backtest -docker run --rm -v $(pwd)/config.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy +docker run --rm -v $(pwd)/config_bittrex.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy if [ $? -ne 0 ]; then echo "failed running backtest" diff --git a/config.json.example b/config_bittrex.json.example similarity index 100% rename from config.json.example rename to config_bittrex.json.example diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index b8829b80f..187b2e3c7 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -54,7 +54,7 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: return conf except ValidationError as e: logger.critical( - f"Invalid configuration. See config.json.example. Reason: {e}" + f"Invalid configuration. Reason: {e}" ) raise ValidationError( best_match(Draft4Validator(conf_schema).iter_errors(conf)).message diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 26e0c4a79..2284209a0 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -21,7 +21,7 @@ from tests.conftest_trades import MOCK_TRADE_COUNT def test_setup_utils_configuration(): args = [ - 'list-exchanges', '--config', 'config.json.example', + 'list-exchanges', '--config', 'config_bittrex.json.example', ] config = setup_utils_configuration(get_args(args), RunMode.OTHER) @@ -40,7 +40,7 @@ def test_start_trading_fail(mocker, caplog): exitmock = mocker.patch("freqtrade.worker.Worker.exit", MagicMock()) args = [ 'trade', - '-c', 'config.json.example' + '-c', 'config_bittrex.json.example' ] start_trading(get_args(args)) assert exitmock.call_count == 1 @@ -122,10 +122,10 @@ def test_list_timeframes(mocker, capsys): match=r"This command requires a configured exchange.*"): start_list_timeframes(pargs) - # Test with --config config.json.example + # Test with --config config_bittrex.json.example args = [ "list-timeframes", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', ] start_list_timeframes(get_args(args)) captured = capsys.readouterr() @@ -169,7 +169,7 @@ def test_list_timeframes(mocker, capsys): # Test with --one-column args = [ "list-timeframes", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--one-column", ] start_list_timeframes(get_args(args)) @@ -209,10 +209,10 @@ def test_list_markets(mocker, markets, capsys): match=r"This command requires a configured exchange.*"): start_list_markets(pargs, False) - # Test with --config config.json.example + # Test with --config config_bittrex.json.example args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--print-list", ] start_list_markets(get_args(args), False) @@ -239,7 +239,7 @@ def test_list_markets(mocker, markets, capsys): # Test with --all: all markets args = [ "list-markets", "--all", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--print-list", ] start_list_markets(get_args(args), False) @@ -252,7 +252,7 @@ def test_list_markets(mocker, markets, capsys): # Test list-pairs subcommand: active pairs args = [ "list-pairs", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--print-list", ] start_list_markets(get_args(args), True) @@ -264,7 +264,7 @@ def test_list_markets(mocker, markets, capsys): # Test list-pairs subcommand with --all: all pairs args = [ "list-pairs", "--all", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--print-list", ] start_list_markets(get_args(args), True) @@ -277,7 +277,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, base=ETH, LTC args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--base", "ETH", "LTC", "--print-list", ] @@ -290,7 +290,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, base=LTC args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--base", "LTC", "--print-list", ] @@ -303,7 +303,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, quote=USDT, USD args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--quote", "USDT", "USD", "--print-list", ] @@ -316,7 +316,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, quote=USDT args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--quote", "USDT", "--print-list", ] @@ -329,7 +329,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, base=LTC, quote=USDT args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--base", "LTC", "--quote", "USDT", "--print-list", ] @@ -342,7 +342,7 @@ def test_list_markets(mocker, markets, capsys): # active pairs, base=LTC, quote=USDT args = [ "list-pairs", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--base", "LTC", "--quote", "USD", "--print-list", ] @@ -355,7 +355,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, base=LTC, quote=USDT, NONEXISTENT args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--base", "LTC", "--quote", "USDT", "NONEXISTENT", "--print-list", ] @@ -368,7 +368,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, base=LTC, quote=NONEXISTENT args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--base", "LTC", "--quote", "NONEXISTENT", "--print-list", ] @@ -381,7 +381,7 @@ def test_list_markets(mocker, markets, capsys): # Test tabular output args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', ] start_list_markets(get_args(args), False) captured = capsys.readouterr() @@ -391,7 +391,7 @@ def test_list_markets(mocker, markets, capsys): # Test tabular output, no markets found args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--base", "LTC", "--quote", "NONEXISTENT", ] start_list_markets(get_args(args), False) @@ -403,7 +403,7 @@ def test_list_markets(mocker, markets, capsys): # Test --print-json args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--print-json" ] start_list_markets(get_args(args), False) @@ -415,7 +415,7 @@ def test_list_markets(mocker, markets, capsys): # Test --print-csv args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--print-csv" ] start_list_markets(get_args(args), False) @@ -427,7 +427,7 @@ def test_list_markets(mocker, markets, capsys): # Test --one-column args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--one-column" ] start_list_markets(get_args(args), False) @@ -439,7 +439,7 @@ def test_list_markets(mocker, markets, capsys): # Test --one-column args = [ "list-markets", - '--config', 'config.json.example', + '--config', 'config_bittrex.json.example', "--one-column" ] with pytest.raises(OperationalException, match=r"Cannot get markets.*"): @@ -781,7 +781,7 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): patched_configuration_load_config_file(mocker, default_conf) args = [ 'test-pairlist', - '-c', 'config.json.example' + '-c', 'config_bittrex.json.example' ] start_test_pairlist(get_args(args)) @@ -795,7 +795,7 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): args = [ 'test-pairlist', - '-c', 'config.json.example', + '-c', 'config_bittrex.json.example', '--one-column', ] start_test_pairlist(get_args(args)) @@ -804,7 +804,7 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): args = [ 'test-pairlist', - '-c', 'config.json.example', + '-c', 'config_bittrex.json.example', '--print-json', ] start_test_pairlist(get_args(args)) diff --git a/tests/test_arguments.py b/tests/test_arguments.py index e2a1ae53c..60c2cfbac 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -172,7 +172,7 @@ def test_download_data_options() -> None: def test_plot_dataframe_options() -> None: args = [ 'plot-dataframe', - '-c', 'config.json.example', + '-c', 'config_bittrex.json.example', '--indicators1', 'sma10', 'sma100', '--indicators2', 'macd', 'fastd', 'fastk', '--plot-limit', '30', diff --git a/tests/test_main.py b/tests/test_main.py index f55aea336..70632aeaa 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -67,12 +67,12 @@ def test_main_fatal_exception(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = ['trade', '-c', 'config.json.example'] + args = ['trade', '-c', 'config_bittrex.json.example'] # Test Main + the KeyboardInterrupt exception with pytest.raises(SystemExit): main(args) - assert log_has('Using config: config.json.example ...', caplog) + assert log_has('Using config: config_bittrex.json.example ...', caplog) assert log_has('Fatal exception!', caplog) @@ -85,12 +85,12 @@ def test_main_keyboard_interrupt(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.wallets.Wallets.update', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = ['trade', '-c', 'config.json.example'] + args = ['trade', '-c', 'config_bittrex.json.example'] # Test Main + the KeyboardInterrupt exception with pytest.raises(SystemExit): main(args) - assert log_has('Using config: config.json.example ...', caplog) + assert log_has('Using config: config_bittrex.json.example ...', caplog) assert log_has('SIGINT received, aborting ...', caplog) @@ -106,12 +106,12 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = ['trade', '-c', 'config.json.example'] + args = ['trade', '-c', 'config_bittrex.json.example'] # Test Main + the KeyboardInterrupt exception with pytest.raises(SystemExit): main(args) - assert log_has('Using config: config.json.example ...', caplog) + assert log_has('Using config: config_bittrex.json.example ...', caplog) assert log_has('Oh snap!', caplog) @@ -157,12 +157,12 @@ def test_main_reload_config(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = Arguments(['trade', '-c', 'config.json.example']).get_parsed_arg() + args = Arguments(['trade', '-c', 'config_bittrex.json.example']).get_parsed_arg() worker = Worker(args=args, config=default_conf) with pytest.raises(SystemExit): - main(['trade', '-c', 'config.json.example']) + main(['trade', '-c', 'config_bittrex.json.example']) - assert log_has('Using config: config.json.example ...', caplog) + assert log_has('Using config: config_bittrex.json.example ...', caplog) assert worker_mock.call_count == 4 assert reconfigure_mock.call_count == 1 assert isinstance(worker.freqtrade, FreqtradeBot) @@ -180,7 +180,7 @@ def test_reconfigure(mocker, default_conf) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = Arguments(['trade', '-c', 'config.json.example']).get_parsed_arg() + args = Arguments(['trade', '-c', 'config_bittrex.json.example']).get_parsed_arg() worker = Worker(args=args, config=default_conf) freqtrade = worker.freqtrade diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 8f3ac464e..96c9868a9 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -363,7 +363,7 @@ def test_start_plot_dataframe(mocker): aup = mocker.patch("freqtrade.plot.plotting.load_and_plot_trades", MagicMock()) args = [ "plot-dataframe", - "--config", "config.json.example", + "--config", "config_bittrex.json.example", "--pairs", "ETH/BTC" ] start_plot_dataframe(get_args(args)) @@ -407,7 +407,7 @@ def test_start_plot_profit(mocker): aup = mocker.patch("freqtrade.plot.plotting.plot_profit", MagicMock()) args = [ "plot-profit", - "--config", "config.json.example", + "--config", "config_bittrex.json.example", "--pairs", "ETH/BTC" ] start_plot_profit(get_args(args)) From 16f96753564bbe791a7d4edc2b918a1ec58f92f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jan 2021 20:35:10 +0100 Subject: [PATCH 168/183] Fix whitelist expansion problem --- freqtrade/plugins/pairlist/pairlist_helpers.py | 4 ++-- tests/plugins/test_pairlist.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py index 04320fe16..924bfb293 100644 --- a/freqtrade/plugins/pairlist/pairlist_helpers.py +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -19,7 +19,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], try: comp = re.compile(pair_wc) result_partial = [ - pair for pair in available_pairs if re.match(comp, pair) + pair for pair in available_pairs if re.fullmatch(comp, pair) ] # Add all matching pairs. # If there are no matching pairs (Pair not on exchange) keep it. @@ -35,7 +35,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], try: comp = re.compile(pair_wc) result += [ - pair for pair in available_pairs if re.match(comp, pair) + pair for pair in available_pairs if re.fullmatch(comp, pair) ] except re.error as err: raise ValueError(f"Wildcard error in {pair_wc}, {err}") diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 910a9580c..d62230e76 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -870,6 +870,9 @@ def test_performance_filter(mocker, whitelist_conf, pairlists, pair_allowlist, o (['*UP/USDT', 'BTC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'], None), + (['BTC/USD'], + ['BTC/USD', 'BTC/USDT'], + ['BTC/USD']), ]) def test_expand_pairlist(wildcardlist, pairs, expected): if expected is None: @@ -901,7 +904,11 @@ def test_expand_pairlist(wildcardlist, pairs, expected): (['*UP/USDT', 'BTC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'], None), - (['HELLO/WORLD'], [], ['HELLO/WORLD']) # Invalid pair kept + (['HELLO/WORLD'], [], ['HELLO/WORLD']), # Invalid pair kept + (['BTC/USD'], + ['BTC/USD', 'BTC/USDT'], + ['BTC/USD']), + ]) def test_expand_pairlist_keep_invalid(wildcardlist, pairs, expected): if expected is None: From 9a3c425cf408f2eacc7c16856bfe605ed7caaf0f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Jan 2021 08:53:05 +0100 Subject: [PATCH 169/183] Update slack link --- CONTRIBUTING.md | 2 +- README.md | 2 +- docs/developer.md | 2 +- docs/faq.md | 2 +- docs/index.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c52a8e93..afa41ed33 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ Few pointers for contributions: - New features need to contain unit tests, must conform to PEP8 (max-line-length = 100) and should be documented with the introduction PR. - PR's can be declared as `[WIP]` - which signify Work in Progress Pull Requests (which are not finished). -If you are unsure, discuss the feature on our [discord server](https://discord.gg/MA9v74M), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. +If you are unsure, discuss the feature on our [discord server](https://discord.gg/MA9v74M), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. ## Getting started diff --git a/README.md b/README.md index c61116219..db648198f 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ For any questions not covered by the documentation or for further information ab Please check out our [discord server](https://discord.gg/MA9v74M). -You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA). +You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA). ### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) diff --git a/docs/developer.md b/docs/developer.md index 299f2f77f..831d9d2f8 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -2,7 +2,7 @@ This page is intended for developers of Freqtrade, people who want to contribute to the Freqtrade codebase or documentation, or people who want to understand the source code of the application they're running. -All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/MA9v74M) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA) where you can ask questions. +All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/MA9v74M) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA) where you can ask questions. ## Documentation diff --git a/docs/faq.md b/docs/faq.md index 5742f512a..8a0c61b29 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -143,7 +143,7 @@ freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossD ### Why does it take a long time to run hyperopt? -* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. +* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. * If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers: diff --git a/docs/index.md b/docs/index.md index 38e040d7a..b489861f0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -65,7 +65,7 @@ For any questions not covered by the documentation or for further information ab Please check out our [discord server](https://discord.gg/MA9v74M). -You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA). +You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA). ## Ready to try? From c22cccb55b98a69b536b0b2b129f77a8da41dec7 Mon Sep 17 00:00:00 2001 From: "Tho Pham (Alex)" <20041501+thopd88@users.noreply.github.com> Date: Mon, 25 Jan 2021 12:24:47 +0700 Subject: [PATCH 170/183] Fix operator does not exist: boolean = integer --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 926f22225..db67e9771 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -200,7 +200,7 @@ class FreqtradeBot(LoggingMixin): Notify the user when the bot is stopped and there are still open trades active. """ - open_trades = Trade.get_trades([Trade.is_open == 1]).all() + open_trades = Trade.get_trades([Trade.is_open == True]).all() if len(open_trades) != 0: msg = { From b976baae3fe06267e64ae7809c15849bde6bdd03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 05:37:16 +0000 Subject: [PATCH 171/183] Bump cachetools from 4.2.0 to 4.2.1 Bumps [cachetools](https://github.com/tkem/cachetools) from 4.2.0 to 4.2.1. - [Release notes](https://github.com/tkem/cachetools/releases) - [Changelog](https://github.com/tkem/cachetools/blob/master/CHANGELOG.rst) - [Commits](https://github.com/tkem/cachetools/compare/v4.2.0...v4.2.1) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53e5bec12..268813e02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ aiohttp==3.7.3 SQLAlchemy==1.3.22 python-telegram-bot==13.1 arrow==0.17.0 -cachetools==4.2.0 +cachetools==4.2.1 requests==2.25.1 urllib3==1.26.2 wrapt==1.12.1 From 9422062cbd9456c95798a3b0ef49c27c037e5e3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 05:37:25 +0000 Subject: [PATCH 172/183] Bump blosc from 1.10.1 to 1.10.2 Bumps [blosc](https://github.com/blosc/python-blosc) from 1.10.1 to 1.10.2. - [Release notes](https://github.com/blosc/python-blosc/releases) - [Changelog](https://github.com/Blosc/python-blosc/blob/master/RELEASE_NOTES.rst) - [Commits](https://github.com/blosc/python-blosc/compare/v1.10.1...v1.10.2) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53e5bec12..2fe4c63d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ tabulate==0.8.7 pycoingecko==1.4.0 jinja2==2.11.2 tables==3.6.1 -blosc==1.10.1 +blosc==1.10.2 # find first, C search in arrays py_find_1st==1.1.4 From afdcd2c0afb4ab3745cb853a1219b3393b6a2c11 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 05:37:42 +0000 Subject: [PATCH 173/183] Bump pytest-cov from 2.10.1 to 2.11.1 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.10.1 to 2.11.1. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.10.1...v2.11.1) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 749450289..01066959a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ flake8-tidy-imports==4.2.1 mypy==0.790 pytest==6.2.1 pytest-asyncio==0.14.0 -pytest-cov==2.10.1 +pytest-cov==2.11.1 pytest-mock==3.5.1 pytest-random-order==1.0.4 isort==5.7.0 From d4e9037e6e15e3f380522bf7dc3ec3d85e967002 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 05:37:49 +0000 Subject: [PATCH 174/183] Bump pandas from 1.2.0 to 1.2.1 Bumps [pandas](https://github.com/pandas-dev/pandas) from 1.2.0 to 1.2.1. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/master/RELEASE.md) - [Commits](https://github.com/pandas-dev/pandas/compare/v1.2.0...v1.2.1) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53e5bec12..e19fe683d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy==1.19.5 -pandas==1.2.0 +pandas==1.2.1 ccxt==1.40.74 aiohttp==3.7.3 From fb99cf14599c5a047e70703a493912ccfdfe41a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 05:37:50 +0000 Subject: [PATCH 175/183] Bump prompt-toolkit from 3.0.10 to 3.0.14 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.10 to 3.0.14. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.10...3.0.14) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53e5bec12..cf2dcb0c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,4 +36,4 @@ pyjwt==2.0.1 colorama==0.4.4 # Building config files interactively questionary==1.9.0 -prompt-toolkit==3.0.10 +prompt-toolkit==3.0.14 From cb749b578dc5bd0a6879b95d445e9f071c4064c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 05:37:51 +0000 Subject: [PATCH 176/183] Bump scikit-learn from 0.24.0 to 0.24.1 Bumps [scikit-learn](https://github.com/scikit-learn/scikit-learn) from 0.24.0 to 0.24.1. - [Release notes](https://github.com/scikit-learn/scikit-learn/releases) - [Commits](https://github.com/scikit-learn/scikit-learn/compare/0.24.0...0.24.1) Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index fbb963cf9..104fbf454 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -3,7 +3,7 @@ # Required for hyperopt scipy==1.6.0 -scikit-learn==0.24.0 +scikit-learn==0.24.1 scikit-optimize==0.8.1 filelock==3.0.12 joblib==1.0.0 From f98bd40955190fc388114ab4cc9062a7ef22ec89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 08:24:17 +0000 Subject: [PATCH 177/183] Bump ccxt from 1.40.74 to 1.40.99 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.40.74 to 1.40.99. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst) - [Commits](https://github.com/ccxt/ccxt/compare/1.40.74...1.40.99) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 471650685..cc8861d82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.19.5 pandas==1.2.1 -ccxt==1.40.74 +ccxt==1.40.99 aiohttp==3.7.3 SQLAlchemy==1.3.22 python-telegram-bot==13.1 From 91b6c02947d239f731f39d5b775a94f3549baf5e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Jan 2021 20:57:05 +0100 Subject: [PATCH 178/183] Update download-data --dl-trades sample command --- docs/data-download.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/data-download.md b/docs/data-download.md index 2d77a8a17..4c7376933 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -308,10 +308,13 @@ Since this data is large by default, the files use gzip by default. They are sto To use this mode, simply add `--dl-trades` to your call. This will swap the download method to download trades, and resamples the data locally. +!!! Warning "do not use" + You should not use this unless you're a kraken user. Most other exchanges provide OHLCV data with sufficient history. + Example call: ```bash -freqtrade download-data --exchange binance --pairs XRP/ETH ETH/BTC --days 20 --dl-trades +freqtrade download-data --exchange kraken --pairs XRP/EUR ETH/EUR --days 20 --dl-trades ``` !!! Note From 8f529f48da6c43e2e73d456b39b1e5fe9c645088 Mon Sep 17 00:00:00 2001 From: "Tho Pham (Alex)" <20041501+thopd88@users.noreply.github.com> Date: Tue, 26 Jan 2021 07:38:25 +0700 Subject: [PATCH 179/183] Update freqtrade/freqtradebot.py use is_open.is_(True) Co-authored-by: Matthias --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index db67e9771..f45d4cacc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -200,7 +200,7 @@ class FreqtradeBot(LoggingMixin): Notify the user when the bot is stopped and there are still open trades active. """ - open_trades = Trade.get_trades([Trade.is_open == True]).all() + open_trades = Trade.get_trades([Trade.is_open.is_(True)]).all() if len(open_trades) != 0: msg = { From 5ab8cc56a40bce22e9556ac82507cb9b01ea54eb Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 26 Jan 2021 08:13:43 +0100 Subject: [PATCH 180/183] Update docs to also work for postgres --- docs/strategy-customization.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 688a1c338..7e998570f 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -653,7 +653,7 @@ The following example queries for the current pair and trades from today, howeve if self.config['runmode'].value in ('live', 'dry_run'): trades = Trade.get_trades([Trade.pair == metadata['pair'], Trade.open_date > datetime.utcnow() - timedelta(days=1), - Trade.is_open == False, + Trade.is_open.is_(False), ]).order_by(Trade.close_date).all() # Summarize profit for this pair. curdayprofit = sum(trade.close_profit for trade in trades) @@ -719,7 +719,7 @@ if self.config['runmode'].value in ('live', 'dry_run'): # fetch closed trades for the last 2 days trades = Trade.get_trades([Trade.pair == metadata['pair'], Trade.open_date > datetime.utcnow() - timedelta(days=2), - Trade.is_open == False, + Trade.is_open.is_(False), ]).all() # Analyze the conditions you'd like to lock the pair .... will probably be different for every strategy sumprofit = sum(trade.close_profit for trade in trades) From 4d7f3e570b18d1927ad00514088616c84c627ad7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 26 Jan 2021 17:15:47 +0100 Subject: [PATCH 181/183] Add test for spreadfilter division exception --- tests/plugins/test_pairlist.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index d62230e76..fda2b1409 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -719,6 +719,32 @@ def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, oh assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count +def test_spreadfilter_invalid_data(mocker, default_conf, markets, tickers, caplog): + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'SpreadFilter', 'max_spread_ratio': 0.1}] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + + ftbot = get_patched_freqtradebot(mocker, default_conf) + ftbot.pairlists.refresh_pairlist() + + assert len(ftbot.pairlists.whitelist) == 5 + + tickers.return_value['ETH/BTC']['ask'] = 0.0 + del tickers.return_value['TKN/BTC'] + del tickers.return_value['LTC/BTC'] + mocker.patch.multiple('freqtrade.exchange.Exchange', get_tickers=tickers) + + ftbot.pairlists.refresh_pairlist() + assert log_has_re(r'Removed .* invalid ticker data.*', caplog) + + assert len(ftbot.pairlists.whitelist) == 2 + + @pytest.mark.parametrize("pairlistconfig,desc_expected,exception_expected", [ ({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010, "max_price": 1.0}, From a9b4d6de33a6e103a5c237163e16622e78b3bd5c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 26 Jan 2021 17:16:57 +0100 Subject: [PATCH 182/183] Check for existance of ask key in ticker closes #4267 --- freqtrade/plugins/pairlist/SpreadFilter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py index 2f3fe47e3..9fa211750 100644 --- a/freqtrade/plugins/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -43,7 +43,7 @@ class SpreadFilter(IPairList): :param ticker: ticker dict as returned from ccxt.load_markets() :return: True if the pair can stay, false if it should be removed """ - if 'bid' in ticker and 'ask' in ticker: + if 'bid' in ticker and 'ask' in ticker and ticker['ask']: spread = 1 - ticker['bid'] / ticker['ask'] if spread > self._max_spread_ratio: self.log_once(f"Removed {pair} from whitelist, because spread " @@ -52,4 +52,6 @@ class SpreadFilter(IPairList): return False else: return True + self.log_once(f"Removed {pair} from whitelist due to invalid ticker data: {ticker}", + logger.info) return False From eac98dbbd69954573bea3671aebc91938639f2f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 27 Jan 2021 07:29:40 +0100 Subject: [PATCH 183/183] Version bump to 2021.1 --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index e96e7f530..74c8c412c 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = 'develop' +__version__ = '2021.1' if __version__ == 'develop':