merged upstream

freqtrade/commands/__init__.py

@@ -10,6 +10,7 @@ from freqtrade.commands.arguments import Arguments
 from freqtrade.commands.build_config_commands import start_new_config
 from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,
                                               start_download_data, start_list_data)
+from freqtrade.commands.db_commands import start_convert_db
 from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui,
                                                 start_new_strategy)
 from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show

freqtrade/commands/arguments.py

@@ -82,7 +82,9 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
 ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
                     "trade_source", "timeframe", "plot_auto_open", ]
 
-ARGS_INSTALL_UI = ["erase_ui_only", 'ui_version']
+ARGS_CONVERT_DB = ["db_url", "db_url_from"]
+
+ARGS_INSTALL_UI = ["erase_ui_only", "ui_version"]
 
 ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"]
 
@@ -181,7 +183,7 @@ class Arguments:
         self._build_args(optionlist=['version'], parser=self.parser)
 
         from freqtrade.commands import (start_backtesting, start_backtesting_show,
-                                        start_convert_data, start_convert_trades,
+                                        start_convert_data, start_convert_db, start_convert_trades,
                                         start_create_userdir, start_download_data, start_edge,
                                         start_hyperopt, start_hyperopt_list, start_hyperopt_show,
                                         start_install_ui, start_list_data, start_list_exchanges,
@@ -374,6 +376,14 @@ class Arguments:
         test_pairlist_cmd.set_defaults(func=start_test_pairlist)
         self._build_args(optionlist=ARGS_TEST_PAIRLIST, parser=test_pairlist_cmd)
 
+        # Add db-convert subcommand
+        convert_db = subparsers.add_parser(
+            "convert-db",
+            help="Migrate database to different system",
+        )
+        convert_db.set_defaults(func=start_convert_db)
+        self._build_args(optionlist=ARGS_CONVERT_DB, parser=convert_db)
+
         # Add install-ui subcommand
         install_ui_cmd = subparsers.add_parser(
             'install-ui',

freqtrade/commands/cli_options.py

@@ -106,6 +106,11 @@ AVAILABLE_CLI_OPTIONS = {
         f'`{constants.DEFAULT_DB_DRYRUN_URL}` for Dry Run).',
         metavar='PATH',
     ),
+    "db_url_from": Arg(
+        '--db-url-from',
+        help='Source db url to use when migrating a database.',
+        metavar='PATH',
+    ),
     "sd_notify": Arg(
         '--sd-notify',
         help='Notify systemd service manager.',

freqtrade/commands/db_commands.py

@@ -0,0 +1,55 @@
+import logging
+from typing import Any, Dict
+
+from sqlalchemy import func
+
+from freqtrade.configuration.config_setup import setup_utils_configuration
+from freqtrade.enums.runmode import RunMode
+
+
+logger = logging.getLogger(__name__)
+
+
+def start_convert_db(args: Dict[str, Any]) -> None:
+    from sqlalchemy.orm import make_transient
+
+    from freqtrade.persistence import Order, Trade, init_db
+    from freqtrade.persistence.migrations import set_sequence_ids
+    from freqtrade.persistence.pairlock import PairLock
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    init_db(config['db_url'], False)
+    session_target = Trade._session
+    init_db(config['db_url_from'], False)
+    logger.info("Starting db migration.")
+
+    trade_count = 0
+    pairlock_count = 0
+    for trade in Trade.get_trades():
+        trade_count += 1
+        make_transient(trade)
+        for o in trade.orders:
+            make_transient(o)
+
+        session_target.add(trade)
+
+    session_target.commit()
+
+    for pairlock in PairLock.query:
+        pairlock_count += 1
+        make_transient(pairlock)
+        session_target.add(pairlock)
+    session_target.commit()
+
+    # Update sequences
+    max_trade_id = session_target.query(func.max(Trade.id)).scalar()
+    max_order_id = session_target.query(func.max(Order.id)).scalar()
+    max_pairlock_id = session_target.query(func.max(PairLock.id)).scalar()
+
+    set_sequence_ids(session_target.get_bind(),
+                     trade_id=max_trade_id,
+                     order_id=max_order_id,
+                     pairlock_id=max_pairlock_id)
+
+    logger.info(f"Migrated {trade_count} Trades, and {pairlock_count} Pairlocks.")

freqtrade/configuration/configuration.py

@@ -147,6 +147,9 @@ class Configuration:
             config.update({'db_url': self.args['db_url']})
             logger.info('Parameter --db-url detected ...')
 
+        self._args_to_config(config, argname='db_url_from',
+                             logstring='Parameter --db-url-from detected ...')
+
         if config.get('force_entry_enable', False):
             logger.warning('`force_entry_enable` RPC message enabled.')
 

freqtrade/exchange/okx.py

@@ -20,7 +20,7 @@ class Okx(Exchange):
     """
 
     _ft_has: Dict = {
-        "ohlcv_candle_limit": 300,
+        "ohlcv_candle_limit": 100,
         "mark_ohlcv_timeframe": "4h",
         "funding_fee_timeframe": "8h",
     }

freqtrade/persistence/__init__.py

@@ -1,5 +1,5 @@
 # flake8: noqa: F401
 
-from freqtrade.persistence.models import (LocalTrade, Order, Trade, clean_dry_run_db, cleanup_db,
-                                          init_db)
+from freqtrade.persistence.models import clean_dry_run_db, cleanup_db, init_db
 from freqtrade.persistence.pairlock_middleware import PairLocks
+from freqtrade.persistence.trade_model import LocalTrade, Order, Trade

freqtrade/persistence/base.py

@@ -0,0 +1,7 @@
+
+from typing import Any
+
+from sqlalchemy.orm import declarative_base
+
+
+_DECL_BASE: Any = declarative_base()

freqtrade/persistence/migrations.py

@@ -46,7 +46,7 @@ def get_last_sequence_ids(engine, trade_back_name, order_back_name):
     return order_id, trade_id
 
 
-def set_sequence_ids(engine, order_id, trade_id):
+def set_sequence_ids(engine, order_id, trade_id, pairlock_id=None):
 
     if engine.name == 'postgresql':
         with engine.begin() as connection:
@@ -54,6 +54,9 @@ def set_sequence_ids(engine, order_id, trade_id):
                 connection.execute(text(f"ALTER SEQUENCE orders_id_seq RESTART WITH {order_id}"))
             if trade_id:
                 connection.execute(text(f"ALTER SEQUENCE trades_id_seq RESTART WITH {trade_id}"))
+            if pairlock_id:
+                connection.execute(
+                    text(f"ALTER SEQUENCE pairlocks_id_seq RESTART WITH {pairlock_id}"))
 
 
 def drop_index_on_table(engine, inspector, table_bak_name):
@@ -99,7 +102,10 @@ def migrate_trades_and_orders_table(
     liquidation_price = get_column_def(cols, 'liquidation_price',
                                        get_column_def(cols, 'isolated_liq', 'null'))
     # sqlite does not support literals for booleans
-    is_short = get_column_def(cols, 'is_short', '0')
+    if engine.name == 'postgresql':
+        is_short = get_column_def(cols, 'is_short', 'false')
+    else:
+        is_short = get_column_def(cols, 'is_short', '0')
 
     # Margin Properties
     interest_rate = get_column_def(cols, 'interest_rate', '0.0')

freqtrade/persistence/pairlock.py

@@ -0,0 +1,70 @@
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+
+from sqlalchemy import Boolean, Column, DateTime, Integer, String, or_
+from sqlalchemy.orm import Query
+
+from freqtrade.constants import DATETIME_PRINT_FORMAT
+from freqtrade.persistence.base import _DECL_BASE
+
+
+class PairLock(_DECL_BASE):
+    """
+    Pair Locks database model.
+    """
+    __tablename__ = 'pairlocks'
+
+    id = Column(Integer, primary_key=True)
+
+    pair = Column(String(25), nullable=False, index=True)
+    # lock direction - long, short or * (for both)
+    side = Column(String(25), nullable=False, default="*")
+    reason = Column(String(255), nullable=True)
+    # Time the pair was locked (start time)
+    lock_time = Column(DateTime, nullable=False)
+    # Time until the pair is locked (end time)
+    lock_end_time = Column(DateTime, nullable=False, index=True)
+
+    active = Column(Boolean, nullable=False, default=True, index=True)
+
+    def __repr__(self):
+        lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT)
+        lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT)
+        return (
+            f'PairLock(id={self.id}, pair={self.pair}, side={self.side}, lock_time={lock_time}, '
+            f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})')
+
+    @staticmethod
+    def query_pair_locks(pair: Optional[str], now: datetime, side: str = '*') -> Query:
+        """
+        Get all currently active locks for this pair
+        :param pair: Pair to check for. Returns all current locks if pair is empty
+        :param now: Datetime object (generated via datetime.now(timezone.utc)).
+        """
+        filters = [PairLock.lock_end_time > now,
+                   # Only active locks
+                   PairLock.active.is_(True), ]
+        if pair:
+            filters.append(PairLock.pair == pair)
+        if side != '*':
+            filters.append(or_(PairLock.side == side, PairLock.side == '*'))
+        else:
+            filters.append(PairLock.side == '*')
+
+        return PairLock.query.filter(
+            *filters
+        )
+
+    def to_json(self) -> Dict[str, Any]:
+        return {
+            'id': self.id,
+            'pair': self.pair,
+            'lock_time': self.lock_time.strftime(DATETIME_PRINT_FORMAT),
+            'lock_timestamp': int(self.lock_time.replace(tzinfo=timezone.utc).timestamp() * 1000),
+            'lock_end_time': self.lock_end_time.strftime(DATETIME_PRINT_FORMAT),
+            'lock_end_timestamp': int(self.lock_end_time.replace(tzinfo=timezone.utc
+                                                                 ).timestamp() * 1000),
+            'reason': self.reason,
+            'side': self.side,
+            'active': self.active,
+        }

freqtrade/rpc/rpc.py

@@ -177,16 +177,19 @@ class RPC:
                         current_rate = NAN
                 else:
                     current_rate = trade.close_rate
-                current_profit = trade.calc_profit_ratio(current_rate)
-                current_profit_abs = trade.calc_profit(current_rate)
-                current_profit_fiat: Optional[float] = None
-                # Calculate fiat profit
-                if self._fiat_converter:
-                    current_profit_fiat = self._fiat_converter.convert_amount(
-                        current_profit_abs,
-                        self._freqtrade.config['stake_currency'],
-                        self._freqtrade.config['fiat_display_currency']
-                    )
+                if len(trade.select_filled_orders(trade.entry_side)) > 0:
+                    current_profit = trade.calc_profit_ratio(current_rate)
+                    current_profit_abs = trade.calc_profit(current_rate)
+                    current_profit_fiat: Optional[float] = None
+                    # Calculate fiat profit
+                    if self._fiat_converter:
+                        current_profit_fiat = self._fiat_converter.convert_amount(
+                            current_profit_abs,
+                            self._freqtrade.config['stake_currency'],
+                            self._freqtrade.config['fiat_display_currency']
+                        )
+                else:
+                    current_profit = current_profit_abs = current_profit_fiat = 0.0
 
                 # Calculate guaranteed profit (in case of trailing stop)
                 stoploss_entry_dist = trade.calc_profit(trade.stop_loss)
@@ -235,8 +238,12 @@ class RPC:
                         trade.pair, side='exit', is_short=trade.is_short, refresh=False)
                 except (PricingError, ExchangeError):
                     current_rate = NAN
-                trade_profit = trade.calc_profit(current_rate)
-                profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}'
+                if len(trade.select_filled_orders(trade.entry_side)) > 0:
+                    trade_profit = trade.calc_profit(current_rate)
+                    profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}'
+                else:
+                    trade_profit = 0.0
+                    profit_str = f'{0.0:.2f}'
                 direction_str = ('S' if trade.is_short else 'L') if nonspot else ''
                 if self._fiat_converter:
                     fiat_profit = self._fiat_converter.convert_amount(
@@ -244,7 +251,7 @@ class RPC:
                         stake_currency,
                         fiat_display_currency
                     )
-                    if fiat_profit and not isnan(fiat_profit):
+                    if not isnan(fiat_profit):
                         profit_str += f" ({fiat_profit:.2f})"
                         fiat_profit_sum = fiat_profit if isnan(fiat_profit_sum) \
                             else fiat_profit_sum + fiat_profit

freqtrade/strategy/interface.py

@@ -15,10 +15,8 @@ from freqtrade.data.dataprovider import DataProvider
 from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, SignalDirection, SignalTagType,
                              SignalType, TradingMode)
 from freqtrade.exceptions import OperationalException, StrategyError
-from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
-from freqtrade.exchange.exchange import timeframe_to_next_date
-from freqtrade.persistence import PairLocks, Trade
-from freqtrade.persistence.models import LocalTrade, Order
+from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date, timeframe_to_seconds
+from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade
 from freqtrade.strategy.hyper import HyperStrategyMixin
 from freqtrade.strategy.informative_decorator import (InformativeData, PopulateIndicators,
                                                       _create_and_merge_informative_pair,

freqtrade/wallets.py

@@ -300,7 +300,7 @@ class Wallets:
 
         if min_stake_amount is not None and min_stake_amount > max_stake_amount:
             if self._log:
-                logger.warning("Minimum stake amount > available balance."
+                logger.warning("Minimum stake amount > available balance. "
                                f"{min_stake_amount} > {max_stake_amount}")
             return 0
         if min_stake_amount is not None and stake_amount < min_stake_amount: