Merge branch 'develop' of https://github.com/freqtrade/freqtrade into max-open-trades

freqtrade/commands/cli_options.py

@@ -633,10 +633,11 @@ AVAILABLE_CLI_OPTIONS = {
               "1: by enter_tag, "
               "2: by enter_tag and exit_tag, "
               "3: by pair and enter_tag, "
-              "4: by pair, enter_ and exit_tag (this can get quite large)"),
+              "4: by pair, enter_ and exit_tag (this can get quite large), "
+              "5: by exit_tag"),
         nargs='+',
         default=['0', '1', '2'],
-        choices=['0', '1', '2', '3', '4'],
+        choices=['0', '1', '2', '3', '4', '5'],
     ),
     "enter_reason_list": Arg(
         "--enter-reason-list",

freqtrade/commands/data_commands.py

@@ -14,6 +14,7 @@ from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import market_is_active, timeframe_to_minutes
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist
 from freqtrade.resolvers import ExchangeResolver
+from freqtrade.util.binance_mig import migrate_binance_futures_data
 
 
 logger = logging.getLogger(__name__)
@@ -86,6 +87,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
                     "Please use `--dl-trades` instead for this exchange "
                     "(will unfortunately take a long time)."
                     )
+            migrate_binance_futures_data(config)
             pairs_not_available = refresh_backtest_ohlcv_data(
                 exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
                 datadir=config['datadir'], timerange=timerange,
@@ -145,6 +147,7 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
     """
     config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
     if ohlcv:
+        migrate_binance_futures_data(config)
         candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', ['spot'])]
         for candle_type in candle_types:
             convert_ohlcv_format(config,

freqtrade/data/entryexitanalysis.py

@@ -141,6 +141,12 @@ def _do_group_table_output(bigdf, glist):
             # 4: profit summaries grouped by pair, enter_ and exit_tag (this can get quite large)
             if g == "4":
                 group_mask = ['pair', 'enter_reason', 'exit_reason']
+
+            # 5: profit summaries grouped by exit_tag
+            if g == "5":
+                group_mask = ['exit_reason']
+                sortcols = ['exit_reason']
+
             if group_mask:
                 new = bigdf.groupby(group_mask).agg(agg_mask).reset_index()
                 new.columns = group_mask + agg_cols

freqtrade/data/history/idatahandler.py

@@ -374,6 +374,21 @@ class IDataHandler(ABC):
                 logger.warning(f"{pair}, {candle_type}, {timeframe}, "
                                f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}")
 
+    def rename_futures_data(
+            self, pair: str, new_pair: str, timeframe: str, candle_type: CandleType):
+        """
+        Temporary method to migrate data from old naming to new naming (BTC/USDT -> BTC/USDT:USDT)
+        Only used for binance to support the binance futures naming unification.
+        """
+
+        file_old = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)
+        file_new = self._pair_data_filename(self._datadir, new_pair, timeframe, candle_type)
+        # print(file_old, file_new)
+        if file_new.exists():
+            logger.warning(f"{file_new} exists already, can't migrate {pair}.")
+            return
+        file_old.rename(file_new)
+
 
 def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
     """

freqtrade/exchange/binance.py

@@ -28,7 +28,7 @@ class Binance(Exchange):
         "trades_pagination": "id",
         "trades_pagination_arg": "fromId",
         "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000],
-        "ccxt_futures_name": "future"
+        "ccxt_futures_name": "swap"
     }
     _ft_has_futures: Dict = {
         "stoploss_order_types": {"limit": "stop", "market": "stop_market"},

freqtrade/freqtradebot.py

@@ -33,6 +33,7 @@ from freqtrade.rpc.external_message_consumer import ExternalMessageConsumer
 from freqtrade.strategy.interface import IStrategy
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from freqtrade.util import FtPrecise
+from freqtrade.util.binance_mig import migrate_binance_futures_names
 from freqtrade.wallets import Wallets
 
 
@@ -177,6 +178,8 @@ class FreqtradeBot(LoggingMixin):
         Called on startup and after reloading the bot - triggers notifications and
         performs startup tasks
         """
+        migrate_binance_futures_names(self.config)
+
         self.rpc.startup_messages(self.config, self.pairlists, self.protections)
         # Update older trades with precision and precision mode
         self.startup_backpopulate_precision()

freqtrade/optimize/backtesting.py

@@ -37,6 +37,7 @@ from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.strategy.interface import IStrategy
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
+from freqtrade.util.binance_mig import migrate_binance_futures_data
 from freqtrade.wallets import Wallets
 
 
@@ -157,6 +158,7 @@ class Backtesting:
         self._can_short = self.trading_mode != TradingMode.SPOT
         self._position_stacking: bool = self.config.get('position_stacking', False)
         self.enable_protections: bool = self.config.get('enable_protections', False)
+        migrate_binance_futures_data(config)
 
         self.init_backtest()
 

freqtrade/resolvers/iresolver.py

@@ -89,7 +89,8 @@ class IResolver:
             module = importlib.util.module_from_spec(spec)
             try:
                 spec.loader.exec_module(module)  # type: ignore # importlib does not use typehints
-            except (ModuleNotFoundError, SyntaxError, ImportError, NameError) as err:
+            except (AttributeError, ModuleNotFoundError, SyntaxError,
+                    ImportError, NameError) as err:
                 # Catch errors in case a specific module is not installed
                 logger.warning(f"Could not import {module_path} due to '{err}'")
                 if enum_failed:

freqtrade/util/binance_mig.py

@@ -0,0 +1,78 @@
+import logging
+
+from packaging import version
+
+from freqtrade.constants import Config
+from freqtrade.enums.tradingmode import TradingMode
+from freqtrade.exceptions import OperationalException
+from freqtrade.persistence.pairlock import PairLock
+from freqtrade.persistence.trade_model import Trade
+
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_binance_futures_names(config: Config):
+
+    if (
+        not (config.get('trading_mode', TradingMode.SPOT) == TradingMode.FUTURES
+             and config['exchange']['name'] == 'binance')
+    ):
+        # only act on new futures
+        return
+    import ccxt
+    if version.parse("2.6.6") > version.parse(ccxt.__version__):
+        raise OperationalException(
+            "Please follow the update instructions in the docs "
+            "(https://www.freqtrade.io/en/latest/updating/) to install a compatible ccxt version.")
+    _migrate_binance_futures_db(config)
+    migrate_binance_futures_data(config)
+
+
+def _migrate_binance_futures_db(config: Config):
+    logger.warning('Migrating binance futures pairs in database.')
+    trades = Trade.get_trades([Trade.exchange == 'binance', Trade.trading_mode == 'FUTURES']).all()
+    for trade in trades:
+        if ':' in trade.pair:
+            # already migrated
+            continue
+        new_pair = f"{trade.pair}:{trade.stake_currency}"
+        trade.pair = new_pair
+
+        for order in trade.orders:
+            order.ft_pair = new_pair
+            # Should symbol be migrated too?
+            # order.symbol = new_pair
+    Trade.commit()
+    pls = PairLock.query.filter(PairLock.pair.notlike('%:%'))
+    for pl in pls:
+        pl.pair = f"{pl.pair}:{config['stake_currency']}"
+    # print(pls)
+    # pls.update({'pair': concat(PairLock.pair,':USDT')})
+    Trade.commit()
+    logger.warning('Done migrating binance futures pairs in database.')
+
+
+def migrate_binance_futures_data(config: Config):
+
+    if (
+        not (config.get('trading_mode', TradingMode.SPOT) == TradingMode.FUTURES
+             and config['exchange']['name'] == 'binance')
+    ):
+        # only act on new futures
+        return
+
+    from freqtrade.data.history.idatahandler import get_datahandler
+    dhc = get_datahandler(config['datadir'], config.get('dataformat_ohlcv', 'json'))
+
+    paircombs = dhc.ohlcv_get_available_data(
+        config['datadir'],
+        config.get('trading_mode', TradingMode.SPOT)
+        )
+
+    for pair, timeframe, candle_type in paircombs:
+        if ':' in pair:
+            # already migrated
+            continue
+        new_pair = f"{pair}:{config['stake_currency']}"
+        dhc.rename_futures_data(pair, new_pair, timeframe, candle_type)