From 86ba7dae92467118e225f4e23c9c00c6600f28ac Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 7 Jan 2023 08:56:40 +0900 Subject: [PATCH 01/10] change sharpe hyperopt loss --- .../hyperopt_loss/hyperopt_loss_sharpe.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py index 2c8ae552d..0db14adab 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py @@ -22,25 +22,13 @@ class SharpeHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function(results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, - *args, **kwargs) -> float: + config: Config, *args, **kwargs) -> float: """ Objective function, returns smaller number for more optimal results. Uses Sharpe Ratio calculation. """ - total_profit = results["profit_ratio"] - days_period = (max_date - min_date).days - - # adding slippage of 0.1% per trade - total_profit = total_profit - 0.0005 - expected_returns_mean = total_profit.sum() / days_period - up_stdev = np.std(total_profit) - - if up_stdev != 0: - sharp_ratio = expected_returns_mean / up_stdev * np.sqrt(365) - else: - # Define high (negative) sharpe ratio to be clear that this is NOT optimal. - sharp_ratio = -20. - + starting_balance = config['dry_run_wallet'] + sharp_ratio = calculate_sharpe(results, min_date, max_date, starting_balance) # print(expected_returns_mean, up_stdev, sharp_ratio) return -sharp_ratio From 157bf962f76102ae9d2aeecac57683c8802b58dd Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 7 Jan 2023 09:14:56 +0900 Subject: [PATCH 02/10] add missing imports --- freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py index 0db14adab..aff4e5787 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py @@ -10,7 +10,8 @@ import numpy as np from pandas import DataFrame from freqtrade.optimize.hyperopt import IHyperOptLoss - +from freqtrade.constants import Config +from freqtrade.data.metrics import calculate_sharpe class SharpeHyperOptLoss(IHyperOptLoss): """ From d3b1aa7f01247d14f4bb7a571a1f66ddacbddb49 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 7 Jan 2023 09:19:06 +0900 Subject: [PATCH 03/10] update sortino calc --- .../hyperopt_loss/hyperopt_loss_sortino.py | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py index b231370dd..1d9914f7f 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py @@ -10,7 +10,8 @@ import numpy as np from pandas import DataFrame from freqtrade.optimize.hyperopt import IHyperOptLoss - +from freqtrade.constants import Config +from freqtrade.data.metrics import calculate_sortino class SortinoHyperOptLoss(IHyperOptLoss): """ @@ -22,28 +23,13 @@ class SortinoHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function(results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, - *args, **kwargs) -> float: + config: Config, *args, **kwargs) -> float: """ Objective function, returns smaller number for more optimal results. Uses Sortino Ratio calculation. """ - total_profit = results["profit_ratio"] - days_period = (max_date - min_date).days - - # adding slippage of 0.1% per trade - total_profit = total_profit - 0.0005 - expected_returns_mean = total_profit.sum() / days_period - - results['downside_returns'] = 0 - results.loc[total_profit < 0, 'downside_returns'] = results['profit_ratio'] - down_stdev = np.std(results['downside_returns']) - - if down_stdev != 0: - sortino_ratio = expected_returns_mean / down_stdev * np.sqrt(365) - else: - # Define high (negative) sortino ratio to be clear that this is NOT optimal. - sortino_ratio = -20. - + starting_balance = config['dry_run_wallet'] + sortino_ratio = calculate_sortino(results, min_date, max_date, starting_balance) # print(expected_returns_mean, down_stdev, sortino_ratio) return -sortino_ratio From 6198b21001eceac1ac18a9eef729ef4873c02c2b Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 7 Jan 2023 09:30:16 +0900 Subject: [PATCH 04/10] update calmar loss --- .../hyperopt_loss/hyperopt_loss_calmar.py | 39 +++---------------- 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py index 2b591824f..1f7f8488f 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py @@ -11,7 +11,7 @@ from typing import Any, Dict from pandas import DataFrame from freqtrade.constants import Config -from freqtrade.data.metrics import calculate_max_drawdown +from freqtrade.data.metrics import calculate_calmar from freqtrade.optimize.hyperopt import IHyperOptLoss @@ -23,42 +23,15 @@ class CalmarHyperOptLoss(IHyperOptLoss): """ @staticmethod - def hyperopt_loss_function( - results: DataFrame, - trade_count: int, - min_date: datetime, - max_date: datetime, - config: Config, - processed: Dict[str, DataFrame], - backtest_stats: Dict[str, Any], - *args, - **kwargs - ) -> float: + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + config: Config, *args, **kwargs) -> float: """ Objective function, returns smaller number for more optimal results. Uses Calmar Ratio calculation. """ - total_profit = backtest_stats["profit_total"] - days_period = (max_date - min_date).days - - # adding slippage of 0.1% per trade - total_profit = total_profit - 0.0005 - expected_returns_mean = total_profit.sum() / days_period * 100 - - # calculate max drawdown - try: - _, _, _, _, _, max_drawdown = calculate_max_drawdown( - results, value_col="profit_abs" - ) - except ValueError: - max_drawdown = 0 - - if max_drawdown != 0: - calmar_ratio = expected_returns_mean / max_drawdown * msqrt(365) - else: - # Define high (negative) calmar ratio to be clear that this is NOT optimal. - calmar_ratio = -20.0 - + starting_balance = config['dry_run_wallet'] + calmar_ratio = calculate_calmar(results, min_date, max_date, starting_balance) # print(expected_returns_mean, max_drawdown, calmar_ratio) return -calmar_ratio From c1042996db480af774d327e617f482ad0a098680 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sat, 7 Jan 2023 09:46:46 +0900 Subject: [PATCH 05/10] flake8 fix --- freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py | 2 -- freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py | 2 +- freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py index 1f7f8488f..b8935b08e 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py @@ -5,8 +5,6 @@ This module defines the alternative HyperOptLoss class which can be used for Hyperoptimization. """ from datetime import datetime -from math import sqrt as msqrt -from typing import Any, Dict from pandas import DataFrame diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py index aff4e5787..f6798b69a 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py @@ -6,13 +6,13 @@ Hyperoptimization. """ from datetime import datetime -import numpy as np from pandas import DataFrame from freqtrade.optimize.hyperopt import IHyperOptLoss from freqtrade.constants import Config from freqtrade.data.metrics import calculate_sharpe + class SharpeHyperOptLoss(IHyperOptLoss): """ Defines the loss function for hyperopt. diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py index 1d9914f7f..64a332e9a 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py @@ -6,13 +6,13 @@ Hyperoptimization. """ from datetime import datetime -import numpy as np from pandas import DataFrame from freqtrade.optimize.hyperopt import IHyperOptLoss from freqtrade.constants import Config from freqtrade.data.metrics import calculate_sortino + class SortinoHyperOptLoss(IHyperOptLoss): """ Defines the loss function for hyperopt. From 7bf531c8b89b2b81670fc468b47fffb89112e69e Mon Sep 17 00:00:00 2001 From: root Date: Sat, 7 Jan 2023 09:50:05 +0900 Subject: [PATCH 06/10] isort fix --- freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py | 2 +- freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py index f6798b69a..8ebb90fc5 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py @@ -8,9 +8,9 @@ from datetime import datetime from pandas import DataFrame -from freqtrade.optimize.hyperopt import IHyperOptLoss from freqtrade.constants import Config from freqtrade.data.metrics import calculate_sharpe +from freqtrade.optimize.hyperopt import IHyperOptLoss class SharpeHyperOptLoss(IHyperOptLoss): diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py index 64a332e9a..a0122a0bf 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py @@ -8,9 +8,9 @@ from datetime import datetime from pandas import DataFrame -from freqtrade.optimize.hyperopt import IHyperOptLoss from freqtrade.constants import Config from freqtrade.data.metrics import calculate_sortino +from freqtrade.optimize.hyperopt import IHyperOptLoss class SortinoHyperOptLoss(IHyperOptLoss): From c7f485687f98aca1d2af5474c37fad76262dd64b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Jan 2023 15:13:22 +0100 Subject: [PATCH 07/10] Fix ccxt test failure as identified and analyzed https://github.com/ccxt/ccxt/issues/16335 --- freqtrade/exchange/exchange.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b4b5f8342..c72f9479d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -474,7 +474,7 @@ class Exchange: try: if self._api_async: self.loop.run_until_complete( - self._api_async.load_markets(reload=reload)) + self._api_async.load_markets(reload=reload, params={})) except (asyncio.TimeoutError, ccxt.BaseError) as e: logger.warning('Could not load async markets. Reason: %s', e) @@ -483,7 +483,7 @@ class Exchange: def _load_markets(self) -> None: """ Initialize markets both sync and async """ try: - self._markets = self._api.load_markets() + self._markets = self._api.load_markets(params={}) self._load_async_markets() self._last_markets_refresh = arrow.utcnow().int_timestamp if self._ft_has['needs_trading_fees']: @@ -501,7 +501,7 @@ class Exchange: return None logger.debug("Performing scheduled market reload..") try: - self._markets = self._api.load_markets(reload=True) + self._markets = self._api.load_markets(reload=True, params={}) # Also reload async markets to avoid issues with newly listed pairs self._load_async_markets(reload=True) self._last_markets_refresh = arrow.utcnow().int_timestamp @@ -1705,7 +1705,7 @@ class Exchange: return self._config['fee'] # validate that markets are loaded before trying to get fee if self._api.markets is None or len(self._api.markets) == 0: - self._api.load_markets() + self._api.load_markets(params={}) return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, price=price, takerOrMaker=taker_or_maker)['rate'] From 1d5440ff71f15882eb19d21e755c20faff7fb43b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Jan 2023 14:19:48 +0000 Subject: [PATCH 08/10] Bump ccxt from 2.4.60 to 2.5.46 Bumps [ccxt](https://github.com/ccxt/ccxt) from 2.4.60 to 2.5.46. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/2.4.60...2.5.46) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b132971df..3c12db49b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.24.1 pandas==1.5.2 pandas-ta==0.3.14b -ccxt==2.4.60 +ccxt==2.5.46 # Pin cryptography for now due to rust build errors with piwheels cryptography==38.0.1; platform_machine == 'armv7l' cryptography==38.0.4; platform_machine != 'armv7l' From 34dbe9deaa8c012c999681b694824d1094804cd4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 Jan 2023 10:08:54 +0100 Subject: [PATCH 09/10] Improve fixture fake results --- tests/optimize/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py index 3d50f37dd..4d257addc 100644 --- a/tests/optimize/conftest.py +++ b/tests/optimize/conftest.py @@ -48,8 +48,8 @@ def hyperopt_results(): return pd.DataFrame( { 'pair': ['ETH/USDT', 'ETH/USDT', 'ETH/USDT', 'ETH/USDT'], - 'profit_ratio': [-0.1, 0.2, -0.1, 0.3], - 'profit_abs': [-0.2, 0.4, -0.2, 0.6], + 'profit_ratio': [-0.1, 0.2, -0.12, 0.3], + 'profit_abs': [-0.2, 0.4, -0.21, 0.6], 'trade_duration': [10, 30, 10, 10], 'amount': [0.1, 0.1, 0.1, 0.1], 'exit_reason': [ExitType.STOP_LOSS, ExitType.ROI, ExitType.STOP_LOSS, ExitType.ROI], From 550ab2b8e85290f11f961ffac12491ab076cf4e1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 8 Jan 2023 11:24:04 +0100 Subject: [PATCH 10/10] Improve select_order to only consider filled where needed. --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/persistence/trade_model.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 258a45008..659eb2660 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -374,7 +374,7 @@ class FreqtradeBot(LoggingMixin): for trade in trades: if not trade.is_open and not trade.fee_updated(trade.exit_side): # Get sell fee - order = trade.select_order(trade.exit_side, False) + order = trade.select_order(trade.exit_side, False, only_filled=True) if not order: order = trade.select_order('stoploss', False) if order: @@ -390,7 +390,7 @@ class FreqtradeBot(LoggingMixin): for trade in trades: with self._exit_lock: if trade.is_open and not trade.fee_updated(trade.entry_side): - order = trade.select_order(trade.entry_side, False) + order = trade.select_order(trade.entry_side, False, only_filled=True) open_order = trade.select_order(trade.entry_side, True) if order and open_order is None: logger.info( diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 0c36d2378..f19b3808f 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -956,11 +956,12 @@ class LocalTrade(): return None def select_order(self, order_side: Optional[str] = None, - is_open: Optional[bool] = None) -> Optional[Order]: + is_open: Optional[bool] = None, only_filled: bool = False) -> Optional[Order]: """ Finds latest order for this orderside and status :param order_side: ft_order_side of the order (either 'buy', 'sell' or 'stoploss') :param is_open: Only search for open orders? + :param only_filled: Only search for Filled orders (only valid with is_open=False). :return: latest Order object if it exists, else None """ orders = self.orders @@ -968,6 +969,8 @@ class LocalTrade(): orders = [o for o in orders if o.ft_order_side == order_side] if is_open is not None: orders = [o for o in orders if o.ft_is_open == is_open] + if is_open is False and only_filled: + orders = [o for o in orders if o.filled and o.status in NON_OPEN_EXCHANGE_STATES] if len(orders) > 0: return orders[-1] else: