Merge branch 'develop' of github.com:lolongcovas/freqtrade into feat/freqai

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
         additional_dependencies:
           - types-cachetools==5.2.1
           - types-filelock==3.2.7
-          - types-requests==2.28.8
+          - types-requests==2.28.9
           - types-tabulate==0.8.11
           - types-python-dateutil==2.8.19
         # stages: [push]

docs/data-download.md

@@ -186,7 +186,7 @@ Freqtrade currently supports 3 data-formats for both OHLCV and trades data:
 By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data.
 
 This can be changed via the `--data-format-ohlcv` and `--data-format-trades` command line arguments respectively.
-To persist this change, you can should also add the following snippet to your configuration, so you don't have to insert the above arguments each time:
+To persist this change, you should also add the following snippet to your configuration, so you don't have to insert the above arguments each time:
 
 ``` jsonc
     // ...

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.3.7
 mkdocs==1.3.1
-mkdocs-material==8.4.0
+mkdocs-material==8.4.1
 mdx_truly_sane_lists==1.3
 pymdown-extensions==9.5
 jinja2==3.1.2

docs/rest-api.md

@@ -317,7 +317,7 @@ whitelist
 ### OpenAPI interface
 
 To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration.
-This will enable the Swagger UI at the `/docs` endpoint. By default, that's running at http://localhost:8080/docs/ - but it'll depend on your settings.
+This will enable the Swagger UI at the `/docs` endpoint. By default, that's running at http://localhost:8080/docs - but it'll depend on your settings.
 
 ### Advanced API usage using JWT tokens
 

docs/strategy-callbacks.md

@@ -75,7 +75,7 @@ class AwesomeStrategy(IStrategy):
 
 ```
 
-### Stake size management
+## Stake size management
 
 Called before entering a trade, makes it possible to manage your position size when placing a new trade.
 

freqtrade/commands/data_commands.py

@@ -81,7 +81,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
                 data_format_trades=config['dataformat_trades'],
             )
         else:
-            if not exchange._ft_has.get('ohlcv_has_history', True):
+            if not exchange.get_option('ohlcv_has_history', True):
                 raise OperationalException(
                     f"Historic klines not available for {exchange.name}. "
                     "Please use `--dl-trades` instead for this exchange "

freqtrade/data/history/history_utils.py

@@ -302,8 +302,8 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
         if trading_mode == 'futures':
             # Predefined candletype (and timeframe) depending on exchange
             # Downloads what is necessary to backtest based on futures data.
-            tf_mark = exchange._ft_has['mark_ohlcv_timeframe']
+            tf_mark = exchange.get_option('mark_ohlcv_timeframe')
-            fr_candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price'])
+            fr_candle_type = CandleType.from_string(exchange.get_option('mark_ohlcv_price'))
             # All exchanges need FundingRate for futures trading.
             # The timeframe is aligned to the mark-price timeframe.
             for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type):
@@ -330,13 +330,12 @@ def _download_trades_history(exchange: Exchange,
     try:
 
         until = None
+        since = 0
         if timerange:
             if timerange.starttype == 'date':
                 since = timerange.startts * 1000
             if timerange.stoptype == 'date':
                 until = timerange.stopts * 1000
-        else:
-            since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000
 
         trades = data_handler.trades_load(pair)
 
@@ -349,6 +348,9 @@ def _download_trades_history(exchange: Exchange,
             logger.info(f"Start earlier than available data. Redownloading trades for {pair}...")
             trades = []
 
+        if not since:
+            since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000
+
         from_id = trades[-1][1] if trades else None
         if trades and since < trades[-1][0]:
             # Reset since to the last available point

freqtrade/exchange/__init__.py

@@ -9,7 +9,8 @@ from freqtrade.exchange.bitpanda import Bitpanda
 from freqtrade.exchange.bittrex import Bittrex
 from freqtrade.exchange.bybit import Bybit
 from freqtrade.exchange.coinbasepro import Coinbasepro
-from freqtrade.exchange.exchange import (amount_to_precision, available_exchanges, ccxt_exchanges,
+from freqtrade.exchange.exchange import (amount_to_contracts, amount_to_precision,
+                                         available_exchanges, ccxt_exchanges, contracts_to_amount,
                                          date_minus_candles, is_exchange_known_ccxt,
                                          is_exchange_officially_supported, market_is_active,
                                          price_to_precision, timeframe_to_minutes,

freqtrade/exchange/exchange.py

@@ -54,8 +54,8 @@ class Exchange:
     # Parameters to add directly to buy/sell calls (like agreeing to trading agreement)
     _params: Dict = {}
 
-    # Additional headers - added to the ccxt object
+    # Additional parameters - added to the ccxt object
-    _headers: Dict = {}
+    _ccxt_params: Dict = {}
 
     # Dict to specify which options each exchange implements
     # This defines defaults, which can be selectively overridden by subclasses using _ft_has
@@ -242,9 +242,9 @@ class Exchange:
         }
         if ccxt_kwargs:
             logger.info('Applying additional ccxt config: %s', ccxt_kwargs)
-        if self._headers:
+        if self._ccxt_params:
-            # Inject static headers after the above output to not confuse users.
+            # Inject static options after the above output to not confuse users.
-            ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs)
+            ccxt_kwargs = deep_merge_dicts(self._ccxt_params, ccxt_kwargs)
         if ccxt_kwargs:
             ex_config.update(ccxt_kwargs)
         try:
@@ -408,7 +408,7 @@ class Exchange:
         else:
             return DataFrame()
 
-    def _get_contract_size(self, pair: str) -> float:
+    def get_contract_size(self, pair: str) -> float:
         if self.trading_mode == TradingMode.FUTURES:
             market = self.markets[pair]
             contract_size: float = 1.0
@@ -421,7 +421,7 @@ class Exchange:
 
     def _trades_contracts_to_amount(self, trades: List) -> List:
         if len(trades) > 0 and 'symbol' in trades[0]:
-            contract_size = self._get_contract_size(trades[0]['symbol'])
+            contract_size = self.get_contract_size(trades[0]['symbol'])
             if contract_size != 1:
                 for trade in trades:
                     trade['amount'] = trade['amount'] * contract_size
@@ -429,7 +429,7 @@ class Exchange:
 
     def _order_contracts_to_amount(self, order: Dict) -> Dict:
         if 'symbol' in order and order['symbol'] is not None:
-            contract_size = self._get_contract_size(order['symbol'])
+            contract_size = self.get_contract_size(order['symbol'])
             if contract_size != 1:
                 for prop in self._ft_has.get('order_props_in_contracts', []):
                     if prop in order and order[prop] is not None:
@@ -438,19 +438,13 @@ class Exchange:
 
     def _amount_to_contracts(self, pair: str, amount: float) -> float:
 
-        contract_size = self._get_contract_size(pair)
+        contract_size = self.get_contract_size(pair)
-        if contract_size and contract_size != 1:
+        return amount_to_contracts(amount, contract_size)
-            return amount / contract_size
-        else:
-            return amount
 
     def _contracts_to_amount(self, pair: str, num_contracts: float) -> float:
 
-        contract_size = self._get_contract_size(pair)
+        contract_size = self.get_contract_size(pair)
-        if contract_size and contract_size != 1:
+        return contracts_to_amount(num_contracts, contract_size)
-            return num_contracts * contract_size
-        else:
-            return num_contracts
 
     def set_sandbox(self, api: ccxt.Exchange, exchange_config: dict, name: str) -> None:
         if exchange_config.get('sandbox'):
@@ -674,6 +668,12 @@ class Exchange:
                 f"Freqtrade does not support {mm_value} {trading_mode.value} on {self.name}"
             )
 
+    def get_option(self, param: str, default: Any = None) -> Any:
+        """
+        Get parameter value from _ft_has
+        """
+        return self._ft_has.get(param, default)
+
     def exchange_has(self, endpoint: str) -> bool:
         """
         Checks if exchange implements a specific API endpoint.
@@ -2892,6 +2892,33 @@ def market_is_active(market: Dict) -> bool:
     return market.get('active', True) is not False
 
 
+def amount_to_contracts(amount: float, contract_size: Optional[float]) -> float:
+    """
+    Convert amount to contracts.
+    :param amount: amount to convert
+    :param contract_size: contract size - taken from exchange.get_contract_size(pair)
+    :return: num-contracts
+    """
+    if contract_size and contract_size != 1:
+        return amount / contract_size
+    else:
+        return amount
+
+
+def contracts_to_amount(num_contracts: float, contract_size: Optional[float]) -> float:
+    """
+    Takes num-contracts and converts it to contract size
+    :param num_contracts: number of contracts
+    :param contract_size: contract size - taken from exchange.get_contract_size(pair)
+    :return: Amount
+    """
+
+    if contract_size and contract_size != 1:
+        return num_contracts * contract_size
+    else:
+        return num_contracts
+
+
 def amount_to_precision(amount: float, amount_precision: Optional[float],
                         precisionMode: Optional[int]) -> float:
     """

freqtrade/exchange/gateio.py

@@ -25,7 +25,6 @@ class Gateio(Exchange):
 
     _ft_has: Dict = {
         "ohlcv_candle_limit": 1000,
-        "ohlcv_volume_currency": "quote",
         "time_in_force_parameter": "timeInForce",
         "order_time_in_force": ['gtc', 'ioc'],
         "stoploss_order_types": {"limit": "limit"},
@@ -34,7 +33,6 @@ class Gateio(Exchange):
 
     _ft_has_futures: Dict = {
         "needs_trading_fees": True,
-        "ohlcv_volume_currency": "base",
         "fee_cost_in_contracts": False,  # Set explicitly to false for clarity
         "order_props_in_contracts": ['amount', 'filled', 'remaining'],
     }

freqtrade/exchange/okx.py

@@ -39,6 +39,8 @@ class Okx(Exchange):
 
     net_only = True
 
+    _ccxt_params: Dict = {'options': {'brokerId': 'ffb5405ad327SUDE'}}
+
     def ohlcv_candle_limit(
             self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int:
         """

freqtrade/freqai/data_drawer.py

@@ -421,7 +421,7 @@ class FreqaiDataDrawer:
             )
 
         # if self.live:
-        self.model_dictionary[dk.model_filename] = model
+        self.model_dictionary[coin] = model
         self.pair_dict[coin]["model_filename"] = dk.model_filename
         self.pair_dict[coin]["data_path"] = str(dk.data_path)
         self.save_drawer_to_disk()
@@ -460,8 +460,8 @@ class FreqaiDataDrawer:
         )
 
         # try to access model in memory instead of loading object from disk to save time
-        if dk.live and dk.model_filename in self.model_dictionary:
+        if dk.live and coin in self.model_dictionary:
-            model = self.model_dictionary[dk.model_filename]
+            model = self.model_dictionary[coin]
         elif not dk.keras:
             model = load(dk.data_path / f"{dk.model_filename}_model.joblib")
         else:

freqtrade/freqai/data_kitchen.py

@@ -601,6 +601,8 @@ class FreqaiDataKitchen:
         is an outlier.
         """
 
+        from math import cos, sin
+
         if predict:
             train_ft_df = self.data_dictionary['train_features']
             pred_ft_df = self.data_dictionary['prediction_features']
@@ -619,23 +621,47 @@ class FreqaiDataKitchen:
 
         else:
 
+            def normalise_distances(distances):
+                normalised_distances = (distances - distances.min()) / \
+                                        (distances.max() - distances.min())
+                return normalised_distances
+
+            def rotate_point(origin, point, angle):
+                # rotate a point counterclockwise by a given angle (in radians)
+                # around a given origin
+                x = origin[0] + cos(angle) * (point[0] - origin[0]) - \
+                                    sin(angle) * (point[1] - origin[1])
+                y = origin[1] + sin(angle) * (point[0] - origin[0]) + \
+                    cos(angle) * (point[1] - origin[1])
+                return (x, y)
+
             MinPts = len(self.data_dictionary['train_features'].columns) * 2
             # measure pairwise distances to train_features.shape[1]*2 nearest neighbours
             neighbors = NearestNeighbors(
                 n_neighbors=MinPts, n_jobs=self.thread_count)
             neighbors_fit = neighbors.fit(self.data_dictionary['train_features'])
             distances, _ = neighbors_fit.kneighbors(self.data_dictionary['train_features'])
-            distances = np.sort(distances, axis=0)
+            distances = np.sort(distances, axis=0).mean(axis=1)
-            index_ten_pct = int(len(distances[:, 1]) * 0.1)
+
-            distances = distances[index_ten_pct:, 1]
+            normalised_distances = normalise_distances(distances)
-            epsilon = distances[-1]
+            x_range = np.linspace(0, 1, len(distances))
+            line = np.linspace(normalised_distances[0],
+                               normalised_distances[-1], len(normalised_distances))
+            deflection = np.abs(normalised_distances - line)
+            max_deflection_loc = np.where(deflection == deflection.max())[0][0]
+            origin = x_range[max_deflection_loc], line[max_deflection_loc]
+            point = x_range[max_deflection_loc], normalised_distances[max_deflection_loc]
+            rot_angle = np.pi / 4
+            elbow_loc = rotate_point(origin, point, rot_angle)
+
+            epsilon = elbow_loc[1] * (distances[-1] - distances[0]) + distances[0]
 
             clustering = DBSCAN(eps=epsilon, min_samples=MinPts,
                                 n_jobs=int(self.thread_count)).fit(
                                                     self.data_dictionary['train_features']
                                                 )
 
-            logger.info(f'DBSCAN found eps of {epsilon}.')
+            logger.info(f'DBSCAN found eps of {epsilon:.2f}.')
 
             self.data['DBSCAN_eps'] = epsilon
             self.data['DBSCAN_min_samples'] = MinPts
@@ -806,7 +832,7 @@ class FreqaiDataKitchen:
 
         if (len(do_predict) - do_predict.sum()) > 0:
             logger.info(
-                f"DI tossed {len(do_predict) - do_predict.sum():.2f} predictions for "
+                f"DI tossed {len(do_predict) - do_predict.sum()} predictions for "
                 "being too far from training data"
             )
 
@@ -981,13 +1007,6 @@ class FreqaiDataKitchen:
             data_load_timerange.stopts = int(time)
             retrain = True
 
-        # logger.info(
-        #     f"downloading data for "
-        #     f"{(data_load_timerange.stopts-data_load_timerange.startts)/SECONDS_IN_DAY:.2f} "
-        #     " days. "
-        #     f"Extension of {additional_seconds/SECONDS_IN_DAY:.2f} days"
-        # )
-
         return retrain, trained_timerange, data_load_timerange
 
     def set_new_model_names(self, pair: str, trained_timerange: TimeRange):

freqtrade/freqai/freqai_interface.py

@@ -82,12 +82,15 @@ class IFreqaiModel(ABC):
         if self.ft_params.get("inlier_metric_window", 0):
             self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2
         self.pair_it = 0
+        self.pair_it_train = 0
         self.total_pairs = len(self.config.get("exchange", {}).get("pair_whitelist"))
         self.last_trade_database_summary: DataFrame = {}
         self.current_trade_database_summary: DataFrame = {}
         self.analysis_lock = Lock()
         self.inference_time: float = 0
+        self.train_time: float = 0
         self.begin_time: float = 0
+        self.begin_time_train: float = 0
         self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe'])
 
     def assert_config(self, config: Dict[str, Any]) -> None:
@@ -130,11 +133,20 @@ class IFreqaiModel(ABC):
             dk = self.start_backtesting(dataframe, metadata, self.dk)
 
         dataframe = dk.remove_features_from_df(dk.return_dataframe)
-        del dk
+        self.clean_up()
         if self.live:
             self.inference_timer('stop')
         return dataframe
 
+    def clean_up(self):
+        """
+        Objects that should be handled by GC already between coins, but
+        are explicitly shown here to help demonstrate the non-persistence of these
+        objects.
+        """
+        self.model = None
+        self.dk = None
+
     @threaded
     def start_scanning(self, strategy: IStrategy) -> None:
         """
@@ -161,9 +173,11 @@ class IFreqaiModel(ABC):
                 dk.set_paths(pair, new_trained_timerange.stopts)
 
                 if retrain:
+                    self.train_timer('start')
                     self.train_model_in_series(
                         new_trained_timerange, pair, strategy, dk, data_load_timerange
                     )
+                    self.train_timer('stop')
 
             self.dd.save_historic_predictions_to_disk()
 
@@ -490,8 +504,7 @@ class IFreqaiModel(ABC):
         data_load_timerange: TimeRange,
     ):
         """
-        Retrieve data and train model in single threaded mode (only used if model directory is empty
+        Retrieve data and train model.
-        upon startup for dry/live )
         :param new_trained_timerange: TimeRange = the timerange to train the model on
         :param metadata: dict = strategy provided metadata
         :param strategy: IStrategy = user defined strategy object
@@ -622,6 +635,24 @@ class IFreqaiModel(ABC):
                 self.inference_time = 0
         return
 
+    def train_timer(self, do='start'):
+        """
+        Timer designed to track the cumulative time spent training the full pairlist in
+        FreqAI.
+        """
+        if do == 'start':
+            self.pair_it_train += 1
+            self.begin_time_train = time.time()
+        elif do == 'stop':
+            end = time.time()
+            self.train_time += (end - self.begin_time_train)
+            if self.pair_it_train == self.total_pairs:
+                logger.info(
+                    f'Total time spent training pairlist {self.train_time:.2f} seconds')
+                self.pair_it_train = 0
+                self.train_time = 0
+        return
+
     # Following methods which are overridden by user made prediction models.
     # See freqai/prediction_models/CatboostPredictionModel.py for an example.
 

freqtrade/freqtradebot.py

@@ -271,7 +271,7 @@ class FreqtradeBot(LoggingMixin):
         Return the number of free open trades slots or 0 if
         max number of open trades reached
         """
-        open_trades = len(Trade.get_open_trades())
+        open_trades = Trade.get_open_trade_count()
         return max(0, self.config['max_open_trades'] - open_trades)
 
     def update_funding_fees(self):
@@ -290,13 +290,14 @@ class FreqtradeBot(LoggingMixin):
 
     def startup_backpopulate_precision(self):
 
-        trades = Trade.get_trades([Trade.precision_mode.is_(None)])
+        trades = Trade.get_trades([Trade.contract_size.is_(None)])
         for trade in trades:
             if trade.exchange != self.exchange.id:
                 continue
             trade.precision_mode = self.exchange.precisionMode
             trade.amount_precision = self.exchange.get_precision_amount(trade.pair)
             trade.price_precision = self.exchange.get_precision_price(trade.pair)
+            trade.contract_size = self.exchange.get_contract_size(trade.pair)
         Trade.commit()
 
     def startup_update_open_orders(self):
@@ -755,6 +756,7 @@ class FreqtradeBot(LoggingMixin):
                 amount_precision=self.exchange.get_precision_amount(pair),
                 price_precision=self.exchange.get_precision_price(pair),
                 precision_mode=self.exchange.precisionMode,
+                contract_size=self.exchange.get_contract_size(pair),
             )
         else:
             # This is additional buy, we reset fee_open_currency so timeout checking can work

freqtrade/optimize/backtesting.py

@@ -24,7 +24,8 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType
                              TradingMode)
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
-from freqtrade.exchange.exchange import amount_to_precision
+from freqtrade.exchange.exchange import (amount_to_contracts, amount_to_precision,
+                                         contracts_to_amount)
 from freqtrade.mixins import LoggingMixin
 from freqtrade.optimize.backtest_caching import get_strategy_run_id
 from freqtrade.optimize.bt_progress import BTProgress
@@ -267,7 +268,7 @@ class Backtesting:
             funding_rates_dict = history.load_data(
                 datadir=self.config['datadir'],
                 pairs=self.pairlists.whitelist,
-                timeframe=self.exchange._ft_has['mark_ohlcv_timeframe'],
+                timeframe=self.exchange.get_option('mark_ohlcv_timeframe'),
                 timerange=self.timerange,
                 startup_candles=0,
                 fail_without_data=True,
@@ -279,12 +280,12 @@ class Backtesting:
             mark_rates_dict = history.load_data(
                 datadir=self.config['datadir'],
                 pairs=self.pairlists.whitelist,
-                timeframe=self.exchange._ft_has['mark_ohlcv_timeframe'],
+                timeframe=self.exchange.get_option('mark_ohlcv_timeframe'),
                 timerange=self.timerange,
                 startup_candles=0,
                 fail_without_data=True,
                 data_format=self.config.get('dataformat_ohlcv', 'json'),
-                candle_type=CandleType.from_string(self.exchange._ft_has["mark_ohlcv_price"])
+                candle_type=CandleType.from_string(self.exchange.get_option("mark_ohlcv_price"))
             )
             # Combine data to avoid combining the data per trade.
             unavailable_pairs = []
@@ -823,11 +824,13 @@ class Backtesting:
             self.order_id_counter += 1
             base_currency = self.exchange.get_pair_base_currency(pair)
             amount_p = (stake_amount / propose_rate) * leverage
-            amount = self.exchange._contracts_to_amount(
+            contract_size = self.exchange.get_contract_size(pair)
-                pair, amount_to_precision(
+            precision_amount = self.exchange.get_precision_amount(pair)
-                    self.exchange._amount_to_contracts(pair, amount_p),
+            amount = contracts_to_amount(
-                    self.exchange.get_precision_amount(pair), self.precision_mode)
+                amount_to_precision(
-                )
+                    amount_to_contracts(amount_p, contract_size),
+                    precision_amount, self.precision_mode),
+                contract_size)
             # Backcalculate actual stake amount.
             stake_amount = amount * propose_rate / leverage
 
@@ -859,9 +862,10 @@ class Backtesting:
                     trading_mode=self.trading_mode,
                     leverage=leverage,
                     # interest_rate=interest_rate,
-                    amount_precision=self.exchange.get_precision_amount(pair),
+                    amount_precision=precision_amount,
                     price_precision=self.exchange.get_precision_price(pair),
                     precision_mode=self.precision_mode,
+                    contract_size=contract_size,
                     orders=[],
                 )
 

freqtrade/persistence/migrations.py

@@ -133,6 +133,7 @@ def migrate_trades_and_orders_table(
     amount_precision = get_column_def(cols, 'amount_precision', 'null')
     price_precision = get_column_def(cols, 'price_precision', 'null')
     precision_mode = get_column_def(cols, 'precision_mode', 'null')
+    contract_size = get_column_def(cols, 'contract_size', 'null')
 
     # Schema migration necessary
     with engine.begin() as connection:
@@ -161,7 +162,7 @@ def migrate_trades_and_orders_table(
             timeframe, open_trade_value, close_profit_abs,
             trading_mode, leverage, liquidation_price, is_short,
             interest_rate, funding_fees, realized_profit,
-            amount_precision, price_precision, precision_mode
+            amount_precision, price_precision, precision_mode, contract_size
             )
         select id, lower(exchange), pair, {base_currency} base_currency,
             {stake_currency} stake_currency,
@@ -189,7 +190,7 @@ def migrate_trades_and_orders_table(
             {is_short} is_short, {interest_rate} interest_rate,
             {funding_fees} funding_fees, {realized_profit} realized_profit,
             {amount_precision} amount_precision, {price_precision} price_precision,
-            {precision_mode} precision_mode
+            {precision_mode} precision_mode, {contract_size} contract_size
             from {trade_back_name}
             """))
 
@@ -308,7 +309,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
     # if ('orders' not in previous_tables
     # or not has_column(cols_orders, 'stop_price')):
     migrating = False
-    if not has_column(cols_trades, 'precision_mode'):
+    if not has_column(cols_trades, 'contract_size'):
         migrating = True
         logger.info(f"Running database migration for trades - "
                     f"backup: {table_back_name}, {order_table_bak_name}")

freqtrade/persistence/trade_model.py

@@ -15,6 +15,7 @@ from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPE
 from freqtrade.enums import ExitType, TradingMode
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import amount_to_precision, price_to_precision
+from freqtrade.exchange.exchange import amount_to_contracts, contracts_to_amount
 from freqtrade.leverage import interest
 from freqtrade.persistence.base import _DECL_BASE
 from freqtrade.util import FtPrecise
@@ -296,6 +297,7 @@ class LocalTrade():
     amount_precision: Optional[float] = None
     price_precision: Optional[float] = None
     precision_mode: Optional[int] = None
+    contract_size: Optional[float] = None
 
     # Leverage trading properties
     liquidation_price: Optional[float] = None
@@ -623,7 +625,11 @@ class LocalTrade():
             else:
                 logger.warning(
                     f'Got different open_order_id {self.open_order_id} != {order.order_id}')
-            amount_tr = amount_to_precision(self.amount, self.amount_precision, self.precision_mode)
+            amount_tr = contracts_to_amount(
+                amount_to_precision(
+                    amount_to_contracts(self.amount, self.contract_size),
+                    self.amount_precision, self.precision_mode),
+                self.contract_size)
             if isclose(order.safe_amount_after_fee, amount_tr, abs_tol=MATH_CLOSE_PREC):
                 self.close(order.safe_price)
             else:
@@ -1044,6 +1050,16 @@ class LocalTrade():
         """
         return Trade.get_trades_proxy(is_open=True)
 
+    @staticmethod
+    def get_open_trade_count() -> int:
+        """
+        get open trade count
+        """
+        if Trade.use_db:
+            return Trade.query.filter(Trade.is_open.is_(True)).count()
+        else:
+            return len(LocalTrade.trades_open)
+
     @staticmethod
     def stoploss_reinitialization(desired_stoploss):
         """
@@ -1132,6 +1148,7 @@ class Trade(_DECL_BASE, LocalTrade):
     amount_precision = Column(Float, nullable=True)
     price_precision = Column(Float, nullable=True)
     precision_mode = Column(Integer, nullable=True)
+    contract_size = Column(Float, nullable=True)
 
     # Leverage trading properties
     leverage = Column(Float, nullable=True, default=1.0)

freqtrade/plugins/pairlist/VolumePairList.py

@@ -73,7 +73,7 @@ class VolumePairList(IPairList):
 
         if (not self._use_range and not (
                 self._exchange.exchange_has('fetchTickers')
-                and self._exchange._ft_has["tickers_have_quoteVolume"])):
+                and self._exchange.get_option("tickers_have_quoteVolume"))):
             raise OperationalException(
                 "Exchange does not support dynamic whitelist in this configuration. "
                 "Please edit your config and either remove Volumepairlist, "
@@ -193,7 +193,7 @@ class VolumePairList(IPairList):
                     ) in candles else None
                 # in case of candle data calculate typical price and quoteVolume for candle
                 if pair_candles is not None and not pair_candles.empty:
-                    if self._exchange._ft_has["ohlcv_volume_currency"] == "base":
+                    if self._exchange.get_option("ohlcv_volume_currency") == "base":
                         pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
                                                          + pair_candles['close']) / 3
 

freqtrade/resolvers/iresolver.py

@@ -193,7 +193,10 @@ class IResolver:
         :return: List of dicts containing 'name', 'class' and 'location' entries
         """
         logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'")
-        objects = []
+        objects: List[Dict[str, Any]] = []
+        if not directory.is_dir():
+            logger.info(f"'{directory}' is not a directory, skipping.")
+            return objects
         for entry in directory.iterdir():
             if (
                 recursive and entry.is_dir()

requirements-dev.txt

@@ -17,7 +17,7 @@ pytest-mock==3.8.2
 pytest-random-order==1.0.4
 isort==5.10.1
 # For datetime mocking
-time-machine==2.7.1
+time-machine==2.8.1
 
 # Convert jupyter notebooks to markdown documents
 nbconvert==6.5.3
@@ -25,6 +25,6 @@ nbconvert==6.5.3
 # mypy types
 types-cachetools==5.2.1
 types-filelock==3.2.7
-types-requests==2.28.8
+types-requests==2.28.9
 types-tabulate==0.8.11
 types-python-dateutil==2.8.19

requirements.txt

@@ -2,7 +2,7 @@ numpy==1.23.2
 pandas==1.4.3
 pandas-ta==0.3.14b
 
-ccxt==1.92.20
+ccxt==1.92.52
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==37.0.4
 aiohttp==3.8.1
@@ -12,7 +12,7 @@ arrow==1.2.2
 cachetools==4.2.2
 requests==2.28.1
 urllib3==1.26.11
-jsonschema==4.9.1
+jsonschema==4.14.0
 TA-Lib==0.4.24
 technical==1.3.0
 tabulate==0.8.10
@@ -34,7 +34,7 @@ orjson==3.7.12
 sdnotify==0.3.2
 
 # API Server
-fastapi==0.79.0
+fastapi==0.79.1
 uvicorn==0.18.2
 pyjwt==2.4.0
 aiofiles==0.8.0

tests/exchange/test_ccxt_compat.py

@@ -409,14 +409,14 @@ class TestCCXTExchange():
                 assert (isinstance(futures_leverage, float) or isinstance(futures_leverage, int))
                 assert futures_leverage >= 1.0
 
-    def test_ccxt__get_contract_size(self, exchange_futures):
+    def test_ccxt_get_contract_size(self, exchange_futures):
         futures, futures_name = exchange_futures
         if futures:
             futures_pair = EXCHANGES[futures_name].get(
                 'futures_pair',
                 EXCHANGES[futures_name]['pair']
             )
-            contract_size = futures._get_contract_size(futures_pair)
+            contract_size = futures.get_contract_size(futures_pair)
             assert (isinstance(contract_size, float) or isinstance(contract_size, int))
             assert contract_size >= 0.0
 

tests/exchange/test_exchange.py

@@ -181,11 +181,11 @@ def test_init_ccxt_kwargs(default_conf, mocker, caplog):
     assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog)
     assert log_has(asynclogmsg, caplog)
     # Test additional headers case
-    Exchange._headers = {'hello': 'world'}
+    Exchange._ccxt_params = {'hello': 'world'}
     ex = Exchange(conf)
 
     assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog)
-    assert ex._api.headers == {'hello': 'world'}
+    assert ex._api.hello == 'world'
     assert ex._ccxt_config == {}
     Exchange._headers = {}
 
@@ -2352,10 +2352,11 @@ def test_fetch_l2_order_book(default_conf, mocker, order_book_l2, exchange_name)
         order_book = exchange.fetch_l2_order_book(pair='ETH/BTC', limit=val)
         assert api_mock.fetch_l2_order_book.call_args_list[0][0][0] == 'ETH/BTC'
         # Not all exchanges support all limits for orderbook
-        if not exchange._ft_has['l2_limit_range'] or val in exchange._ft_has['l2_limit_range']:
+        if (not exchange.get_option('l2_limit_range')
+                or val in exchange.get_option('l2_limit_range')):
             assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == val
         else:
-            next_limit = exchange.get_next_limit_in_list(val, exchange._ft_has['l2_limit_range'])
+            next_limit = exchange.get_next_limit_in_list(val, exchange.get_option('l2_limit_range'))
             assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == next_limit
 
 
@@ -3311,16 +3312,16 @@ def test_merge_ft_has_dict(default_conf, mocker):
 
     ex = Kraken(default_conf)
     assert ex._ft_has != Exchange._ft_has_default
-    assert ex._ft_has['trades_pagination'] == 'id'
+    assert ex.get_option('trades_pagination') == 'id'
-    assert ex._ft_has['trades_pagination_arg'] == 'since'
+    assert ex.get_option('trades_pagination_arg') == 'since'
 
     # Binance defines different values
     ex = Binance(default_conf)
     assert ex._ft_has != Exchange._ft_has_default
-    assert ex._ft_has['stoploss_on_exchange']
+    assert ex.get_option('stoploss_on_exchange')
-    assert ex._ft_has['order_time_in_force'] == ['gtc', 'fok', 'ioc']
+    assert ex.get_option('order_time_in_force') == ['gtc', 'fok', 'ioc']
-    assert ex._ft_has['trades_pagination'] == 'id'
+    assert ex.get_option('trades_pagination') == 'id'
-    assert ex._ft_has['trades_pagination_arg'] == 'fromId'
+    assert ex.get_option('trades_pagination_arg') == 'fromId'
 
     conf = copy.deepcopy(default_conf)
     conf['exchange']['_ft_has_params'] = {"DeadBeef": 20,
@@ -4287,7 +4288,7 @@ def test__fetch_and_calculate_funding_fees_datetime_called(
     ('XLTCUSDT', 0.01, 'futures'),
     ('ETH/USDT:USDT', 10, 'futures')
 ])
-def test__get_contract_size(mocker, default_conf, pair, expected_size, trading_mode):
+def est__get_contract_size(mocker, default_conf, pair, expected_size, trading_mode):
     api_mock = MagicMock()
     default_conf['trading_mode'] = trading_mode
     default_conf['margin_mode'] = 'isolated'
@@ -4306,7 +4307,7 @@ def test__get_contract_size(mocker, default_conf, pair, expected_size, trading_m
             'contractSize': '10',
         }
     })
-    size = exchange._get_contract_size(pair)
+    size = exchange.get_contract_size(pair)
     assert expected_size == size
 
 
@@ -5145,7 +5146,7 @@ def test_stoploss_contract_size(mocker, default_conf, contract_size, order_amoun
     mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y)
 
     exchange = get_patched_exchange(mocker, default_conf, api_mock)
-    exchange._get_contract_size = MagicMock(return_value=contract_size)
+    exchange.get_contract_size = MagicMock(return_value=contract_size)
 
     api_mock.create_order.reset_mock()
     order = exchange.stoploss(

tests/strategy/test_strategy_loading.py

@@ -48,6 +48,10 @@ def test_search_all_strategies_with_failed():
     assert len([x for x in strategies if x['class'] is not None]) == 9
     assert len([x for x in strategies if x['class'] is None]) == 1
 
+    directory = Path(__file__).parent / "strats_nonexistingdir"
+    strategies = StrategyResolver.search_all_objects(directory, enum_failed=True)
+    assert len(strategies) == 0
+
 
 def test_load_strategy(default_conf, result):
     default_conf.update({'strategy': 'SampleStrategy',

tests/test_freqtradebot.py

@@ -473,8 +473,6 @@ def test_create_trade_no_signal(default_conf_usdt, fee, mocker) -> None:
     freqtrade = FreqtradeBot(default_conf_usdt)
     patch_get_signal(freqtrade, enter_long=False, exit_long=False)
 
-    Trade.query = MagicMock()
-    Trade.query.filter = MagicMock()
     assert not freqtrade.create_trade('ETH/USDT')
 
 

tests/test_persistence.py

@@ -1689,6 +1689,7 @@ def test_get_open(fee, is_short, use_db):
 
     create_mock_trades(fee, is_short, use_db)
     assert len(Trade.get_open_trades()) == 4
+    assert Trade.get_open_trade_count() == 4
 
     Trade.use_db = True
 
@@ -1701,6 +1702,7 @@ def test_get_open_lev(fee, use_db):
 
     create_mock_trades_with_leverage(fee, use_db)
     assert len(Trade.get_open_trades()) == 5
+    assert Trade.get_open_trade_count() == 5
 
     Trade.use_db = True