Merge branch 'develop' into pr/cyberjunky/6615

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pycqa/flake8
+    rev: '4.0.1'
+    hooks:
+    -   id: flake8
+        # stages: [push]
+
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v0.942'
+    hooks:
+    -   id: mypy
+        # stages: [push]
+
+-   repo: https://github.com/pycqa/isort
+    rev: '5.10.1'
+    hooks:
+      - id: isort
+        name: isort (python)
+        # stages: [push]

config_examples/config_full.example.json

@@ -182,6 +182,7 @@
     "disable_dataframe_checks": false,
     "strategy": "SampleStrategy",
     "strategy_path": "user_data/strategies/",
+    "add_config_files": [],
     "dataformat_ohlcv": "json",
     "dataformat_trades": "jsongz"
 }

docs/configuration.md

@@ -53,14 +53,33 @@ FREQTRADE__EXCHANGE__SECRET=<yourExchangeSecret>
 
 Multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream.
 
+You can specify additional configuration files in `add_config_files`. Files specified in this parameter will be loaded and merged with the initial config file. The files are resolved relative to the initial configuration file.
+This is similar to using multiple `--config` parameters, but simpler in usage as you don't have to specify all files for all commands.
+
 !!! Tip "Use multiple configuration files to keep secrets secret"
     You can use a 2nd configuration file containing your secrets. That way you can share your "primary" configuration file, while still keeping your API keys for yourself.
 
+    ``` json title="user_data/config.json"
+    "add_config_files": [
+        "config-private.json"
+    ]
+    ```
+    
+    ``` bash
+    freqtrade trade --config user_data/config.json <...>
+    ```
+
+    The 2nd file should only specify what you intend to override.
+    If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`).
+
+    For one-off commands, you can also use the below syntax by specifying multiple "--config" parameters.
+
     ``` bash
     freqtrade trade --config user_data/config.json --config user_data/config-private.json <...>
     ```
-    The 2nd file should only specify what you intend to override.
+
-    If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`).
+    This is equivalent to the example above - but `config-private.json` is specified as cli argument.
+
 
 ## Configuration parameters
 
@@ -175,6 +194,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. <br> **Datatype:** Boolean
 | `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file. <br> **Datatype:** String
 | `user_data_dir` | Directory containing user data. <br> *Defaults to `./user_data/`*. <br> **Datatype:** String
+| `add_config_files` | Additional config files. These files will be loaded and merged with the current config file. The files are resolved relative to the initial file.<br> *Defaults to `[]`*. <br> **Datatype:** List of strings
 | `dataformat_ohlcv` | Data format to use to store historical candle (OHLCV) data. <br> *Defaults to `json`*. <br> **Datatype:** String
 | `dataformat_trades` | Data format to use to store historical trades data. <br> *Defaults to `jsongz`*. <br> **Datatype:** String
 | `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position). <br> [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.*<br> **Datatype:** Boolean

docs/developer.md

@@ -26,6 +26,9 @@ Alternatively (e.g. if your system is not supported by the setup.sh script), fol
 
 This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`.
 
+Then install the git hook scripts by running `pre-commit install`, so your changes will be verified locally before committing.
+This avoids a lot of waiting for CI already, as some basic formatting checks are done locally on your machine.
+
 Before opening a pull request, please familiarize yourself with our [Contributing Guidelines](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md).
 
 ### Devcontainer setup

docs/plotting.md

@@ -96,7 +96,7 @@ Strategy arguments:
 Example:
 
 ``` bash
-freqtrade plot-dataframe -p BTC/ETH
+freqtrade plot-dataframe -p BTC/ETH --strategy AwesomeStrategy
 ```
 
 The `-p/--pairs` argument can be used to specify pairs you would like to plot.
@@ -107,9 +107,6 @@ The `-p/--pairs` argument can be used to specify pairs you would like to plot.
 Specify custom indicators.
 Use `--indicators1` for the main plot and `--indicators2` for the subplot below (if values are in a different range than prices).
 
-!!! Tip
-    You will almost certainly want to specify a custom strategy! This can be done by adding `-s Classname` / `--strategy ClassName` to the command.
-
 ``` bash
 freqtrade plot-dataframe --strategy AwesomeStrategy -p BTC/ETH --indicators1 sma ema --indicators2 macd
 ```

docs/strategy-callbacks.md

@@ -88,10 +88,11 @@ Allows to define custom exit signals, indicating that specified position should
 
 For example you could implement a 1:2 risk-reward ROI with `custom_exit()`.
 
-Using custom_exit() signals in place of stoploss though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange.
+Using `custom_exit()` signals in place of stoploss though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange.
 
 !!! Note
-    Returning a (none-empty) `string` or `True` from this method is equal to setting exit signal on a candle at specified time. This method is not called when exit signal is set already, or if exit signals are disabled (`use_exit_signal=False` or `exit_profit_only=True` while profit is below `exit_profit_offset`). `string` max length is 64 characters. Exceeding this limit will cause the message to be truncated to 64 characters.
+    Returning a (none-empty) `string` or `True` from this method is equal to setting exit signal on a candle at specified time. This method is not called when exit signal is set already, or if exit signals are disabled (`use_exit_signal=False`). `string` max length is 64 characters. Exceeding this limit will cause the message to be truncated to 64 characters.
+    `custom_exit()` will ignore `exit_profit_only`, and will always be called unless `use_exit_signal=False`, even if there is a new enter signal.
 
 An example of how we can use different indicators depending on the current profit and also exit trades that were open longer than one day:
 

docs/strategy_migration.md

@@ -9,7 +9,7 @@ You can use the quick summary as checklist. Please refer to the detailed section
 
 ## Quick summary / migration checklist
 
-Note : `force_exit`, `force_enter`, `emergency_exit` are changed to `force_exit`, `force_enter`, `emergency_exit` respectively.
+Note : `forcesell`, `forcebuy`, `emergencysell` are changed to `force_exit`, `force_enter`, `emergency_exit` respectively.
 
 * Strategy methods:
   * [`populate_buy_trend()` -> `populate_entry_trend()`](#populate_buy_trend)
@@ -145,6 +145,9 @@ Please refer to the [Strategy documentation](strategy-customization.md#exit-sign
 
 ### `custom_sell`
 
+`custom_sell` has been renamed to `custom_exit`.
+It's now also being called for every iteration, independent of current profit and `exit_profit_only` settings.
+
 ``` python hl_lines="2"
 class AwesomeStrategy(IStrategy):
     def custom_sell(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float,

freqtrade/configuration/load_config.py

@@ -75,19 +75,41 @@ def load_config_file(path: str) -> Dict[str, Any]:
     return config
 
 
-def load_from_files(files: List[str]) -> Dict[str, Any]:
+def load_from_files(files: List[str], base_path: Path = None, level: int = 0) -> Dict[str, Any]:
-
+    """
+    Recursively load configuration files if specified.
+    Sub-files are assumed to be relative to the initial config.
+    """
     config: Dict[str, Any] = {}
+    if level > 5:
+        raise OperationalException("Config loop detected.")
 
     if not files:
         return deepcopy(MINIMAL_CONFIG)
-
+    files_loaded = []
     # We expect here a list of config filenames
-    for path in files:
+    for filename in files:
-        logger.info(f'Using config: {path} ...')
+        logger.info(f'Using config: {filename} ...')
-        # Merge config options, overwriting old values
+        if filename == '-':
-        config = deep_merge_dicts(load_config_file(path), config)
+            # Immediately load stdin and return
+            return load_config_file(filename)
+        file = Path(filename)
+        if base_path:
+            # Prepend basepath to allow for relative assignments
+            file = base_path / file
 
-    config['config_files'] = files
+        config_tmp = load_config_file(str(file))
+        if 'add_config_files' in config_tmp:
+            config_sub = load_from_files(
+                config_tmp['add_config_files'], file.resolve().parent, level + 1)
+            files_loaded.extend(config_sub.get('config_files', []))
+            deep_merge_dicts(config_sub, config_tmp)
+
+        files_loaded.insert(0, str(file))
+
+        # Merge config options, overwriting prior values
+        config = deep_merge_dicts(config_tmp, config)
+
+    config['config_files'] = files_loaded
 
     return config

freqtrade/constants.py

@@ -91,15 +91,14 @@ SUPPORTED_FIAT = [
 ]
 
 MINIMAL_CONFIG = {
-    'stake_currency': '',
+    "stake_currency": "",
-    'dry_run': True,
+    "dry_run": True,
-    'exchange': {
+    "exchange": {
-        'name': '',
+        "name": "",
-        'key': '',
+        "key": "",
-        'secret': '',
+        "secret": "",
-        'pair_whitelist': [],
+        "pair_whitelist": [],
-        'ccxt_async_config': {
+        "ccxt_async_config": {
-            'enableRateLimit': True,
         }
     }
 }

freqtrade/exchange/exchange.py

@@ -341,15 +341,11 @@ class Exchange:
         return sorted(set([x['quote'] for _, x in markets.items()]))
 
     def get_pair_quote_currency(self, pair: str) -> str:
-        """
+        """ Return a pair's quote currency (base/quote:settlement) """
-        Return a pair's quote currency
-        """
         return self.markets.get(pair, {}).get('quote', '')
 
     def get_pair_base_currency(self, pair: str) -> str:
-        """
+        """ Return a pair's base currency (base/quote:settlement) """
-        Return a pair's base currency
-        """
         return self.markets.get(pair, {}).get('base', '')
 
     def market_is_future(self, market: Dict[str, Any]) -> bool:

freqtrade/freqtradebot.py

@@ -598,6 +598,7 @@ class FreqtradeBot(LoggingMixin):
             pair, price, stake_amount, trade_side, enter_tag, trade)
 
         if not stake_amount:
+            logger.info(f"No stake amount to enter a trade for {pair}.")
             return False
 
         if pos_adjust:
@@ -675,6 +676,7 @@ class FreqtradeBot(LoggingMixin):
 
         # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL
         fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker')
+        base_currency = self.exchange.get_pair_base_currency(pair)
         open_date = datetime.now(timezone.utc)
         funding_fees = self.exchange.get_funding_fees(
             pair=pair, amount=amount, is_short=is_short, open_date=open_date)
@@ -682,6 +684,8 @@ class FreqtradeBot(LoggingMixin):
         if trade is None:
             trade = Trade(
                 pair=pair,
+                base_currency=base_currency,
+                stake_currency=self.config['stake_currency'],
                 stake_amount=stake_amount,
                 amount=amount,
                 is_open=True,

freqtrade/optimize/backtesting.py

@@ -726,6 +726,7 @@ class Backtesting:
 
         if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount):
             self.order_id_counter += 1
+            base_currency = self.exchange.get_pair_base_currency(pair)
             amount = round((stake_amount / propose_rate) * leverage, 8)
             is_short = (direction == 'short')
             # Necessary for Margin trading. Disabled until support is enabled.
@@ -738,6 +739,8 @@ class Backtesting:
                     id=self.trade_id_counter,
                     open_order_id=self.order_id_counter,
                     pair=pair,
+                    base_currency=base_currency,
+                    stake_currency=self.config['stake_currency'],
                     open_rate=propose_rate,
                     open_rate_requested=propose_rate,
                     open_date=current_time,

freqtrade/persistence/migrations.py

@@ -3,6 +3,8 @@ from typing import List
 
 from sqlalchemy import inspect, text
 
+from freqtrade.exceptions import OperationalException
+
 
 logger = logging.getLogger(__name__)
 
@@ -58,6 +60,8 @@ def migrate_trades_and_orders_table(
         decl_base, inspector, engine,
         trade_back_name: str, cols: List,
         order_back_name: str, cols_order: List):
+    base_currency = get_column_def(cols, 'base_currency', 'null')
+    stake_currency = get_column_def(cols, 'stake_currency', 'null')
     fee_open = get_column_def(cols, 'fee_open', 'fee')
     fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
     fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
@@ -130,7 +134,7 @@ def migrate_trades_and_orders_table(
     # Copy data back - following the correct schema
     with engine.begin() as connection:
         connection.execute(text(f"""insert into trades
-            (id, exchange, pair, is_open,
+            (id, exchange, pair, base_currency, stake_currency, is_open,
             fee_open, fee_open_cost, fee_open_currency,
             fee_close, fee_close_cost, fee_close_currency, open_rate,
             open_rate_requested, close_rate, close_rate_requested, close_profit,
@@ -142,7 +146,8 @@ def migrate_trades_and_orders_table(
             trading_mode, leverage, liquidation_price, is_short,
             interest_rate, funding_fees
             )
-        select id, lower(exchange), pair,
+        select id, lower(exchange), pair, {base_currency} base_currency,
+            {stake_currency} stake_currency,
             is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
             {fee_open_currency} fee_open_currency, {fee_close} fee_close,
             {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
@@ -173,23 +178,6 @@ def migrate_trades_and_orders_table(
     set_sequence_ids(engine, order_id, trade_id)
 
 
-def migrate_open_orders_to_trades(engine):
-    with engine.begin() as connection:
-        connection.execute(text("""
-        insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open)
-        select id ft_trade_id, pair ft_pair, open_order_id,
-            case when close_rate_requested is null then 'buy'
-            else 'sell' end ft_order_side, 1 ft_is_open
-        from trades
-        where open_order_id is not null
-        union all
-        select id ft_trade_id, pair ft_pair, stoploss_order_id order_id,
-            'stoploss' ft_order_side, 1 ft_is_open
-        from trades
-        where stoploss_order_id is not null
-        """))
-
-
 def drop_orders_table(engine, table_back_name: str):
     # Drop and recreate orders table as backup
     # This drops foreign keys, too.
@@ -207,7 +195,7 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List):
     # sqlite does not support literals for booleans
     with engine.begin() as connection:
         connection.execute(text(f"""
-            insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
+            insert into orders (id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
             status, symbol, order_type, side, price, amount, filled, average, remaining, cost,
             order_date, order_filled_date, order_update_date, ft_fee_base)
             select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
@@ -230,7 +218,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
     """
     inspector = inspect(engine)
 
-    cols = inspector.get_columns('trades')
+    cols_trades = inspector.get_columns('trades')
     cols_orders = inspector.get_columns('orders')
     tabs = get_table_names_for_table(inspector, 'trades')
     table_back_name = get_backup_name(tabs, 'trades_bak')
@@ -241,13 +229,17 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
     # Migrates both trades and orders table!
     # if ('orders' not in previous_tables
     # or not has_column(cols_orders, 'leverage')):
-    if not has_column(cols, 'exit_order_status'):
+    if not has_column(cols_trades, 'base_currency'):
         logger.info(f"Running database migration for trades - "
                     f"backup: {table_back_name}, {order_table_bak_name}")
         migrate_trades_and_orders_table(
-            decl_base, inspector, engine, table_back_name, cols, order_table_bak_name, cols_orders)
+            decl_base, inspector, engine, table_back_name, cols_trades,
+            order_table_bak_name, cols_orders)
 
     if 'orders' not in previous_tables and 'trades' in previous_tables:
-        logger.info('Moving open orders to Orders table.')
+        raise OperationalException(
-        migrate_open_orders_to_trades(engine)
+            "Your database seems to be very old. "
+            "Please update to freqtrade 2022.3 to migrate this database or "
+            "start with a fresh database.")
+
     set_sqlite_to_wal(engine)

freqtrade/persistence/models.py

@@ -279,6 +279,8 @@ class LocalTrade():
 
     exchange: str = ''
     pair: str = ''
+    base_currency: str = ''
+    stake_currency: str = ''
     is_open: bool = True
     fee_open: float = 0.0
     fee_open_cost: Optional[float] = None
@@ -397,6 +399,26 @@ class LocalTrade():
         else:
             return "long"
 
+    @property
+    def safe_base_currency(self) -> str:
+        """
+        Compatibility layer for asset - which can be empty for old trades.
+        """
+        try:
+            return self.base_currency or self.pair.split('/')[0]
+        except IndexError:
+            return ''
+
+    @property
+    def safe_quote_currency(self) -> str:
+        """
+        Compatibility layer for asset - which can be empty for old trades.
+        """
+        try:
+            return self.stake_currency or self.pair.split('/')[1].split(':')[0]
+        except IndexError:
+            return ''
+
     def __init__(self, **kwargs):
         for key in kwargs:
             setattr(self, key, kwargs[key])
@@ -423,6 +445,8 @@ class LocalTrade():
         return {
             'trade_id': self.id,
             'pair': self.pair,
+            'base_currency': self.safe_base_currency,
+            'quote_currency': self.safe_quote_currency,
             'is_open': self.is_open,
             'exchange': self.exchange,
             'amount': round(self.amount, 8),
@@ -1051,6 +1075,8 @@ class Trade(_DECL_BASE, LocalTrade):
 
     exchange = Column(String(25), nullable=False)
     pair = Column(String(25), nullable=False, index=True)
+    base_currency = Column(String(25), nullable=True)
+    stake_currency = Column(String(25), nullable=True)
     is_open = Column(Boolean, nullable=False, default=True, index=True)
     fee_open = Column(Float, nullable=False, default=0.0)
     fee_open_cost = Column(Float, nullable=True)

freqtrade/rpc/api_server/api_schemas.py

@@ -203,6 +203,8 @@ class OrderSchema(BaseModel):
 class TradeSchema(BaseModel):
     trade_id: int
     pair: str
+    base_currency: str
+    quote_currency: str
     is_open: bool
     is_short: bool
     exchange: str

freqtrade/rpc/rpc.py

@@ -197,7 +197,6 @@ class RPC:
 
                 trade_dict = trade.to_json()
                 trade_dict.update(dict(
-                    base_currency=self._freqtrade.config['stake_currency'],
                     close_profit=trade.close_profit if trade.close_profit is not None else None,
                     current_rate=current_rate,
                     current_profit=current_profit,  # Deprecated
@@ -223,6 +222,7 @@ class RPC:
     def _rpc_status_table(self, stake_currency: str,
                           fiat_display_currency: str) -> Tuple[List, List, float]:
         trades: List[Trade] = Trade.get_open_trades()
+        nonspot = self._config.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT
         if not trades:
             raise RPCException('no active trade')
         else:
@@ -237,7 +237,7 @@ class RPC:
                     current_rate = NAN
                 trade_profit = trade.calc_profit(current_rate)
                 profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}'
-                direction_str = 'S' if trade.is_short else 'L'
+                direction_str = ('S' if trade.is_short else 'L') if nonspot else ''
                 if self._fiat_converter:
                     fiat_profit = self._fiat_converter.convert_amount(
                         trade_profit,
@@ -267,7 +267,11 @@ class RPC:
             if self._fiat_converter:
                 profitcol += " (" + fiat_display_currency + ")"
 
-            columns = ['ID L/S', 'Pair', 'Since', profitcol]
+            columns = [
+                'ID L/S' if nonspot else 'ID',
+                'Pair',
+                'Since',
+                profitcol]
             if self._config.get('position_adjustment_enable', False):
                 columns.append('# Entries')
             return trades_list, columns, fiat_profit_sum
@@ -791,7 +795,7 @@ class RPC:
             trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
             return trade
         else:
-            return None
+            raise RPCException(f'Failed to enter position for {pair}.')
 
     def _rpc_delete(self, trade_id: int) -> Dict[str, Union[str, int]]:
         """

freqtrade/rpc/telegram.py

@@ -509,7 +509,7 @@ class Telegram(RPCHandler):
                             lines.append("*Open Order:* `{open_order}`")
 
                 lines_detail = self._prepare_entry_details(
-                    r['orders'], r['base_currency'], r['is_open'])
+                    r['orders'], r['quote_currency'], r['is_open'])
                 lines.extend(lines_detail if lines_detail else "")
 
                 # Filter empty lines using list-comprehension

freqtrade/strategy/interface.py

@@ -881,14 +881,10 @@ class IStrategy(ABC, HyperStrategyMixin):
         current_rate = rate
         current_profit = trade.calc_profit_ratio(current_rate)
 
-        if (self.exit_profit_only and current_profit <= self.exit_profit_offset):
+        if self.use_exit_signal:
-            # exit_profit_only and profit doesn't reach the offset - ignore sell signal
+            if exit_ and not enter:
-            pass
-        elif self.use_exit_signal and not enter:
-            if exit_:
                 exit_signal = ExitType.EXIT_SIGNAL
             else:
-                trade_type = "exit_short" if trade.is_short else "sell"
                 custom_reason = strategy_safe_wrapper(self.custom_exit, default_retval=False)(
                     pair=trade.pair, trade=trade, current_time=current_time,
                     current_rate=current_rate, current_profit=current_profit)
@@ -896,13 +892,17 @@ class IStrategy(ABC, HyperStrategyMixin):
                     exit_signal = ExitType.CUSTOM_EXIT
                     if isinstance(custom_reason, str):
                         if len(custom_reason) > CUSTOM_EXIT_MAX_LENGTH:
-                            logger.warning(f'Custom {trade_type} reason returned from '
+                            logger.warning(f'Custom exit reason returned from '
                                            f'custom_exit is too long and was trimmed'
                                            f'to {CUSTOM_EXIT_MAX_LENGTH} characters.')
                             custom_reason = custom_reason[:CUSTOM_EXIT_MAX_LENGTH]
                     else:
                         custom_reason = None
-            if exit_signal in (ExitType.CUSTOM_EXIT, ExitType.EXIT_SIGNAL):
+            if (
+                exit_signal == ExitType.CUSTOM_EXIT
+                or (exit_signal == ExitType.EXIT_SIGNAL
+                    and (not self.exit_profit_only or current_profit > self.exit_profit_offset))
+            ):
                 logger.debug(f"{trade.pair} - Sell signal received. "
                              f"exit_type=ExitType.{exit_signal.name}" +
                              (f", custom_reason={custom_reason}" if custom_reason else ""))

requirements-dev.txt

@@ -7,6 +7,7 @@ coveralls==3.3.1
 flake8==4.0.1
 flake8-tidy-imports==4.6.0
 mypy==0.942
+pre-commit==2.18.1
 pytest==7.1.1
 pytest-asyncio==0.18.3
 pytest-cov==3.0.0

setup.sh

@@ -51,6 +51,7 @@ function updateenv() {
     echo "pip install in-progress. Please wait..."
     ${PYTHON} -m pip install --upgrade pip
     read -p "Do you want to install dependencies for dev [y/N]? "
+    dev=$REPLY
     if [[ $REPLY =~ ^[Yy]$ ]]
     then
         REQUIREMENTS=requirements-dev.txt
@@ -88,6 +89,13 @@ function updateenv() {
     fi
     echo "pip install completed"
     echo
+    if [[ $dev =~ ^[Yy]$ ]]; then
+        ${PYTHON} -m pre-commit install
+        if [ $? -ne 0 ]; then
+            echo "Failed installing pre-commit"
+            exit 1
+        fi
+    fi
 }
 
 # Install tab lib

tests/rpc/test_rpc.py

@@ -52,7 +52,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
     assert results[0] == {
         'trade_id': 1,
         'pair': 'ETH/BTC',
-        'base_currency': 'BTC',
+        'base_currency': 'ETH',
+        'quote_currency': 'BTC',
         'open_date': ANY,
         'open_timestamp': ANY,
         'is_open': ANY,
@@ -135,7 +136,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
     assert results[0] == {
         'trade_id': 1,
         'pair': 'ETH/BTC',
-        'base_currency': 'BTC',
+        'base_currency': 'ETH',
+        'quote_currency': 'BTC',
         'open_date': ANY,
         'open_timestamp': ANY,
         'is_open': ANY,
@@ -1230,8 +1232,8 @@ def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open
     patch_get_signal(freqtradebot)
     rpc = RPC(freqtradebot)
     pair = 'TKN/BTC'
-    trade = rpc._rpc_force_entry(pair, None)
+    with pytest.raises(RPCException, match=r"Failed to enter position for TKN/BTC."):
-    assert trade is None
+        trade = rpc._rpc_force_entry(pair, None)
 
 
 def test_rpc_force_entry_stopped(mocker, default_conf) -> None:

tests/rpc/test_rpc_apiserver.py

@@ -931,6 +931,8 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short,
         'open_order': None,
         'open_rate': 0.123,
         'pair': 'ETH/BTC',
+        'base_currency': 'ETH',
+        'quote_currency': 'BTC',
         'stake_amount': 0.001,
         'stop_loss_abs': ANY,
         'stop_loss_pct': ANY,
@@ -1097,7 +1099,7 @@ def test_api_force_entry(botclient, mocker, fee, endpoint):
 
     # Test creating trade
     fbuy_mock = MagicMock(return_value=Trade(
-        pair='ETH/ETH',
+        pair='ETH/BTC',
         amount=1,
         amount_requested=1,
         exchange='binance',
@@ -1130,7 +1132,9 @@ def test_api_force_entry(botclient, mocker, fee, endpoint):
         'open_date': ANY,
         'open_timestamp': ANY,
         'open_rate': 0.245441,
-        'pair': 'ETH/ETH',
+        'pair': 'ETH/BTC',
+        'base_currency': 'ETH',
+        'quote_currency': 'BTC',
         'stake_amount': 1,
         'stop_loss_abs': None,
         'stop_loss_pct': None,

tests/rpc/test_rpc_telegram.py

@@ -184,7 +184,8 @@ def test_telegram_status(default_conf, update, mocker) -> None:
         _rpc_trade_status=MagicMock(return_value=[{
             'trade_id': 1,
             'pair': 'ETH/BTC',
-            'base_currency': 'BTC',
+            'base_currency': 'ETH',
+            'quote_currency': 'BTC',
             'open_date': arrow.utcnow(),
             'close_date': None,
             'open_rate': 1.099e-05,
@@ -398,8 +399,8 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None:
     fields = re.sub('[ ]+', ' ', line[2].strip()).split(' ')
 
     assert int(fields[0]) == 1
-    assert 'L' in fields[1]
+    # assert 'L' in fields[1]
-    assert 'ETH/BTC' in fields[2]
+    assert 'ETH/BTC' in fields[1]
     assert msg_mock.call_count == 1
 
 
@@ -1253,7 +1254,7 @@ def test_force_exit_no_pair(default_conf, update, ticker, fee, mocker) -> None:
     assert reduce(lambda acc, x: acc + len(x), keyboard, 0) == 5
     assert keyboard[-1][0].text == "Cancel"
 
-    assert keyboard[1][0].callback_data == 'force_exit__2 L'
+    assert keyboard[1][0].callback_data == 'force_exit__2 '
     update = MagicMock()
     update.callback_query = MagicMock()
     update.callback_query.data = keyboard[1][0].callback_data

tests/strategy/test_interface.py

@@ -523,7 +523,7 @@ def test_custom_exit(default_conf, fee, caplog) -> None:
     assert res.exit_type == ExitType.CUSTOM_EXIT
     assert res.exit_flag is True
     assert res.exit_reason == 'h' * 64
-    assert log_has_re('Custom sell reason returned from custom_exit is too long.*', caplog)
+    assert log_has_re('Custom exit reason returned from custom_exit is too long.*', caplog)
 
 
 @pytest.mark.parametrize('side', TRADE_SIDES)

tests/test_configuration.py

@@ -18,7 +18,8 @@ from freqtrade.configuration.deprecated_settings import (check_conflicting_setti
                                                          process_removed_setting,
                                                          process_temporary_deprecated_settings)
 from freqtrade.configuration.environment_vars import flat_vars_to_nested_dict
-from freqtrade.configuration.load_config import load_config_file, load_file, log_config_error_range
+from freqtrade.configuration.load_config import (load_config_file, load_file, load_from_files,
+                                                 log_config_error_range)
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL, ENV_VAR_PREFIX
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
@@ -206,6 +207,32 @@ def test_from_config(default_conf, mocker, caplog) -> None:
     assert isinstance(validated_conf['user_data_dir'], Path)
 
 
+def test_from_recursive_files(testdatadir) -> None:
+    files = testdatadir / "testconfigs/testconfig.json"
+
+    conf = Configuration.from_files([files])
+
+    assert conf
+    # Exchange comes from "the first config"
+    assert conf['exchange']
+    # Pricing comes from the 2nd config
+    assert conf['entry_pricing']
+    assert conf['entry_pricing']['price_side'] == "same"
+    assert conf['exit_pricing']
+    # The other key comes from pricing2, which is imported by pricing.json
+    assert conf['exit_pricing']['price_side'] == "other"
+
+    assert len(conf['config_files']) == 4
+    assert 'testconfig.json' in conf['config_files'][0]
+    assert 'test_pricing_conf.json' in conf['config_files'][1]
+    assert 'test_base_config.json' in conf['config_files'][2]
+    assert 'test_pricing2_conf.json' in conf['config_files'][3]
+
+    files = testdatadir / "testconfigs/recursive.json"
+    with pytest.raises(OperationalException, match="Config loop detected."):
+        load_from_files([files])
+
+
 def test_print_config(default_conf, mocker, caplog) -> None:
     conf1 = deepcopy(default_conf)
     # Delete non-json elements from default_conf

tests/test_freqtradebot.py

@@ -3663,6 +3663,7 @@ def test_exit_profit_only(
     })
     freqtrade = FreqtradeBot(default_conf_usdt)
     patch_get_signal(freqtrade, enter_short=is_short, enter_long=not is_short)
+    freqtrade.strategy.custom_exit = MagicMock(return_value=None)
     if exit_type == ExitType.EXIT_SIGNAL.value:
         freqtrade.strategy.min_roi_reached = MagicMock(return_value=False)
     else:
@@ -3671,10 +3672,15 @@ def test_exit_profit_only(
     freqtrade.enter_positions()
 
     trade = Trade.query.first()
-    trade.is_short = is_short
+    assert trade.is_short == is_short
     oobj = Order.parse_from_ccxt_object(limit_order[eside], limit_order[eside]['symbol'], eside)
     trade.update_trade(oobj)
     freqtrade.wallets.update()
+    if profit_only:
+        assert freqtrade.handle_trade(trade) is False
+        # Custom-exit is called
+        freqtrade.strategy.custom_exit.call_count == 1
+
     patch_get_signal(freqtrade, enter_long=False, exit_short=is_short, exit_long=not is_short)
     assert freqtrade.handle_trade(trade) is handle_first
 

tests/test_persistence.py

@@ -1209,6 +1209,27 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
                                 PRIMARY KEY (id),
                                 CHECK (is_open IN (0, 1))
                                 );"""
+    create_table_order = """CREATE TABLE orders (
+                                id INTEGER NOT NULL,
+                                ft_trade_id INTEGER,
+                                ft_order_side VARCHAR(25) NOT NULL,
+                                ft_pair VARCHAR(25) NOT NULL,
+                                ft_is_open BOOLEAN NOT NULL,
+                                order_id VARCHAR(255) NOT NULL,
+                                status VARCHAR(255),
+                                symbol VARCHAR(25),
+                                order_type VARCHAR(50),
+                                side VARCHAR(25),
+                                price FLOAT,
+                                amount FLOAT,
+                                filled FLOAT,
+                                remaining FLOAT,
+                                cost FLOAT,
+                                order_date DATETIME,
+                                order_filled_date DATETIME,
+                                order_update_date DATETIME,
+                                PRIMARY KEY (id)
+                            );"""
     insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee,
                           open_rate, stake_amount, amount, open_date,
                           stop_loss, initial_stop_loss, max_rate, ticker_interval,
@@ -1222,15 +1243,66 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
                                      stake=default_conf.get("stake_amount"),
                                      amount=amount
                                      )
+    insert_orders = f"""
+        insert into orders (
+            ft_trade_id,
+            ft_order_side,
+            ft_pair,
+            ft_is_open,
+            order_id,
+            status,
+            symbol,
+            order_type,
+            side,
+            price,
+            amount,
+            filled,
+            remaining,
+            cost)
+        values (
+            1,
+            'buy',
+            'ETC/BTC',
+            0,
+            'buy_order',
+            'closed',
+            'ETC/BTC',
+            'limit',
+            'buy',
+            0.00258580,
+            {amount},
+            {amount},
+            0,
+            {amount * 0.00258580}
+        ),
+        (
+            1,
+            'stoploss',
+            'ETC/BTC',
+            0,
+            'stop_order_id222',
+            'closed',
+            'ETC/BTC',
+            'limit',
+            'sell',
+            0.00258580,
+            {amount},
+            {amount},
+            0,
+            {amount * 0.00258580}
+        )
+    """
     engine = create_engine('sqlite://')
     mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine)
 
     # Create table using the old format
     with engine.begin() as connection:
         connection.execute(text(create_table_old))
+        connection.execute(text(create_table_order))
         connection.execute(text("create index ix_trades_is_open on trades(is_open)"))
         connection.execute(text("create index ix_trades_pair on trades(pair)"))
         connection.execute(text(insert_table_old))
+        connection.execute(text(insert_orders))
 
         # fake previous backup
         connection.execute(text("create table trades_bak as select * from trades"))
@@ -1267,8 +1339,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert trade.open_trade_value == trade._calc_open_trade_value()
     assert trade.close_profit_abs is None
 
-    assert log_has("Moving open orders to Orders table.", caplog)
+    orders = trade.orders
-    orders = Order.query.all()
     assert len(orders) == 2
     assert orders[0].order_id == 'buy_order'
     assert orders[0].ft_order_side == 'buy'
@@ -1277,7 +1348,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert orders[1].ft_order_side == 'stoploss'
 
 
-def test_migrate_mid_state(mocker, default_conf, fee, caplog):
+def test_migrate_too_old(mocker, default_conf, fee, caplog):
     """
     Test Database migration (starting with new pairformat)
     """
@@ -1301,6 +1372,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
                                 PRIMARY KEY (id),
                                 CHECK (is_open IN (0, 1))
                                 );"""
+
     insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close,
                           open_rate, stake_amount, amount, open_date)
                           VALUES ('binance', 'ETC/BTC', 1, {fee}, {fee},
@@ -1319,26 +1391,8 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
         connection.execute(text(insert_table_old))
 
     # Run init to test migration
-    init_db(default_conf['db_url'], default_conf['dry_run'])
+    with pytest.raises(OperationalException, match=r'Your database seems to be very old'):
-
+        init_db(default_conf['db_url'], default_conf['dry_run'])
-    assert len(Trade.query.filter(Trade.id == 1).all()) == 1
-    trade = Trade.query.filter(Trade.id == 1).first()
-    assert trade.fee_open == fee.return_value
-    assert trade.fee_close == fee.return_value
-    assert trade.open_rate_requested is None
-    assert trade.close_rate_requested is None
-    assert trade.is_open == 1
-    assert trade.amount == amount
-    assert trade.stake_amount == default_conf.get("stake_amount")
-    assert trade.pair == "ETC/BTC"
-    assert trade.exchange == "binance"
-    assert trade.max_rate == 0.0
-    assert trade.stop_loss == 0.0
-    assert trade.initial_stop_loss == 0.0
-    assert trade.open_trade_value == trade._calc_open_trade_value()
-    assert log_has("trying trades_bak0", caplog)
-    assert log_has("Running database migration for trades - backup: trades_bak0, orders_bak0",
-                   caplog)
 
 
 def test_migrate_get_last_sequence_ids():
@@ -1561,6 +1615,8 @@ def test_to_json(fee):
 
     assert result == {'trade_id': None,
                       'pair': 'ADA/USDT',
+                      'base_currency': 'ADA',
+                      'quote_currency': 'USDT',
                       'is_open': None,
                       'open_date': trade.open_date.strftime("%Y-%m-%d %H:%M:%S"),
                       'open_timestamp': int(trade.open_date.timestamp() * 1000),
@@ -1637,6 +1693,8 @@ def test_to_json(fee):
 
     assert result == {'trade_id': None,
                       'pair': 'XRP/BTC',
+                      'base_currency': 'XRP',
+                      'quote_currency': 'BTC',
                       'open_date': trade.open_date.strftime("%Y-%m-%d %H:%M:%S"),
                       'open_timestamp': int(trade.open_date.timestamp() * 1000),
                       'close_date': trade.close_date.strftime("%Y-%m-%d %H:%M:%S"),

tests/testdata/testconfigs/recursive.json

@@ -0,0 +1,6 @@
+{
+    // This file fails as it's loading itself over and over
+    "add_config_files": [
+        "./recursive.json"
+    ]
+}

tests/testdata/testconfigs/test_base_config.json

@@ -0,0 +1,12 @@
+{
+    "stake_currency": "",
+    "dry_run": true,
+    "exchange": {
+        "name": "",
+        "key": "",
+        "secret": "",
+        "pair_whitelist": [],
+        "ccxt_async_config": {
+        }
+    }
+}

tests/testdata/testconfigs/test_pricing2_conf.json

@@ -0,0 +1,18 @@
+{
+    "entry_pricing": {
+        "price_side": "same",
+        "use_order_book": true,
+        "order_book_top": 1,
+        "price_last_balance": 0.0,
+        "check_depth_of_market": {
+            "enabled": false,
+            "bids_to_ask_delta": 1
+        }
+    },
+    "exit_pricing":{
+        "price_side": "other",
+        "use_order_book": true,
+        "order_book_top": 1,
+        "price_last_balance": 0.0
+    }
+}

tests/testdata/testconfigs/test_pricing_conf.json

@@ -0,0 +1,21 @@
+{
+    "entry_pricing": {
+        "price_side": "same",
+        "use_order_book": true,
+        "order_book_top": 1,
+        "price_last_balance": 0.0,
+        "check_depth_of_market": {
+            "enabled": false,
+            "bids_to_ask_delta": 1
+        }
+    },
+    "exit_pricing":{
+        "price_side": "same",
+        "use_order_book": true,
+        "order_book_top": 1,
+        "price_last_balance": 0.0
+    },
+    "add_config_files": [
+        "./test_pricing2_conf.json"
+    ]
+}

tests/testdata/testconfigs/testconfig.json

@@ -0,0 +1,6 @@
+{
+    "add_config_files": [
+        "test_base_config.json",
+        "test_pricing_conf.json"
+    ]
+}