Merge branch 'develop' of github.com:lolongcovas/freqtrade into feat/freqai

build_helpers/publish_docker_arm64.sh

@@ -6,10 +6,12 @@ export DOCKER_BUILDKIT=1
 # Replace / with _ to create a valid tag
 TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
 TAG_PLOT=${TAG}_plot
+TAG_FREQAI=${TAG}_freqai
 TAG_PI="${TAG}_pi"
 
 TAG_ARM=${TAG}_arm
 TAG_PLOT_ARM=${TAG_PLOT}_arm
+TAG_FREQAI_ARM=${TAG_FREQAI}_arm
 CACHE_IMAGE=freqtradeorg/freqtrade_cache
 
 echo "Running for ${TAG}"
@@ -38,8 +40,10 @@ fi
 docker tag freqtrade:$TAG_ARM ${CACHE_IMAGE}:$TAG_ARM
 
 docker build --cache-from freqtrade:${TAG_ARM} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_PLOT_ARM} -f docker/Dockerfile.plot .
+docker build --cache-from freqtrade:${TAG_ARM} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_FREQAI_ARM} -f docker/Dockerfile.freqai .
 
 docker tag freqtrade:$TAG_PLOT_ARM ${CACHE_IMAGE}:$TAG_PLOT_ARM
+docker tag freqtrade:$TAG_FREQAI_ARM ${CACHE_IMAGE}:$TAG_FREQAI_ARM
 
 # Run backtest
 docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3
@@ -53,6 +57,7 @@ docker images
 
 # docker push ${IMAGE_NAME}
 docker push ${CACHE_IMAGE}:$TAG_PLOT_ARM
+docker push ${CACHE_IMAGE}:$TAG_FREQAI_ARM
 docker push ${CACHE_IMAGE}:$TAG_ARM
 
 # Create multi-arch image
@@ -66,6 +71,9 @@ docker manifest push -p ${IMAGE_NAME}:${TAG}
 docker manifest create ${IMAGE_NAME}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT_ARM} ${CACHE_IMAGE}:${TAG_PLOT}
 docker manifest push -p ${IMAGE_NAME}:${TAG_PLOT}
 
+docker manifest create ${IMAGE_NAME}:${TAG_FREQAI} ${CACHE_IMAGE}:${TAG_FREQAI_ARM} ${CACHE_IMAGE}:${TAG_FREQAI}
+docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI}
+
 # Tag as latest for develop builds
 if [ "${TAG}" = "develop" ]; then
     docker manifest create ${IMAGE_NAME}:latest ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} ${CACHE_IMAGE}:${TAG}

build_helpers/publish_docker_multi.sh

@@ -5,6 +5,7 @@
 # Replace / with _ to create a valid tag
 TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
 TAG_PLOT=${TAG}_plot
+TAG_FREQAI=${TAG}_freqai
 TAG_PI="${TAG}_pi"
 
 PI_PLATFORM="linux/arm/v7"
@@ -49,8 +50,10 @@ fi
 docker tag freqtrade:$TAG ${CACHE_IMAGE}:$TAG
 
 docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot .
+docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG} -t freqtrade:${TAG_FREQAI} -f docker/Dockerfile.freqai .
 
 docker tag freqtrade:$TAG_PLOT ${CACHE_IMAGE}:$TAG_PLOT
+docker tag freqtrade:$TAG_FREQAI ${CACHE_IMAGE}:$TAG_FREQAI
 
 # Run backtest
 docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3
@@ -64,6 +67,7 @@ docker images
 
 docker push ${CACHE_IMAGE}
 docker push ${CACHE_IMAGE}:$TAG_PLOT
+docker push ${CACHE_IMAGE}:$TAG_FREQAI
 docker push ${CACHE_IMAGE}:$TAG
 
 

config_examples/config_full.example.json

@@ -93,6 +93,7 @@
         "secret": "your_exchange_secret",
         "password": "",
         "log_responses": false,
+        // "unknown_fee_rate": 1,
         "ccxt_config": {},
         "ccxt_async_config": {},
         "pair_whitelist": [

docker/Dockerfile.freqai

@@ -2,16 +2,8 @@ ARG sourceimage=freqtradeorg/freqtrade
 ARG sourcetag=develop
 FROM ${sourceimage}:${sourcetag}
 
-USER root
-
-RUN apt-get install -y libgomp1
-
-USER ftuser
-
 # Install dependencies
 COPY requirements-freqai.txt /freqtrade/
 
 RUN pip install -r requirements-freqai.txt --user --no-cache-dir
-# Temporary step - as the source image will contain the wrong (non-freqai) sourcecode
-COPY --chown=ftuser:ftuser . /freqtrade/
 

docs/freqai.md

@@ -75,6 +75,12 @@ pip install -r requirements-freqai.txt
 !!! Note
     Catboost will not be installed on arm devices (raspberry, Mac M1, ARM based VPS, ...), since Catboost does not provide wheels for this platform.
 
+### Usage with docker
+
+For docker users, a dedicated tag with freqAI dependencies is available as `:freqai`.
+As such - you can replace the image line in your docker-compose file with `image: freqtradeorg/freqtrade:develop_freqai`.
+This image contains the regular freqAI dependencies. Similar to native installs, Catboost will not be available on ARM based devices.
+
 ## Configuring FreqAI
 
 ### Parameter table

docs/strategy-customization.md

@@ -617,9 +617,8 @@ Please always check the mode of operation to select the correct method to get da
 ### *available_pairs*
 
 ``` python
-if self.dp:
+for pair, timeframe in self.dp.available_pairs:
-    for pair, timeframe in self.dp.available_pairs:
+    print(f"available {pair}, {timeframe}")
-        print(f"available {pair}, {timeframe}")
 ```
 
 ### *current_whitelist()*
@@ -630,7 +629,7 @@ The strategy might look something like this:
 
 *Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day RSI to buy and sell.*
 
-Due to the limited available data, it's very difficult to resample `5m` candles into daily candles for use in a 14 day RSI. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least!
+Due to the limited available data, it's very difficult to resample `5m` candles into daily candles for use in a 14 day RSI. Most exchanges limit us to just 500-1000 candles which effectively gives us around 1.74 daily candles. We need 14 days at least!
 
 Since we can't resample the data we will have to use an informative pair; and since the whitelist will be dynamic we don't know which pair(s) to use.
 
@@ -653,10 +652,9 @@ This is where calling `self.dp.current_whitelist()` comes in handy.
 
 ``` python
 # fetch live / historical candle (OHLCV) data for the first informative pair
-if self.dp:
+inf_pair, inf_timeframe = self.informative_pairs()[0]
-    inf_pair, inf_timeframe = self.informative_pairs()[0]
+informative = self.dp.get_pair_dataframe(pair=inf_pair,
-    informative = self.dp.get_pair_dataframe(pair=inf_pair,
+                                            timeframe=inf_timeframe)
-                                             timeframe=inf_timeframe)
 ```
 
 !!! Warning "Warning about backtesting"
@@ -671,10 +669,9 @@ It can also be used in specific callbacks to get the signal that caused the acti
 
 ``` python
 # fetch current dataframe
-if self.dp:
+if self.dp.runmode.value in ('live', 'dry_run'):
-    if self.dp.runmode.value in ('live', 'dry_run'):
+    dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'],
-        dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'],
+                                                                timeframe=self.timeframe)
-                                                                 timeframe=self.timeframe)
 ```
 
 !!! Note "No data available"
@@ -684,11 +681,10 @@ if self.dp:
 ### *orderbook(pair, maximum)*
 
 ``` python
-if self.dp:
+if self.dp.runmode.value in ('live', 'dry_run'):
-    if self.dp.runmode.value in ('live', 'dry_run'):
+    ob = self.dp.orderbook(metadata['pair'], 1)
-        ob = self.dp.orderbook(metadata['pair'], 1)
+    dataframe['best_bid'] = ob['bids'][0][0]
-        dataframe['best_bid'] = ob['bids'][0][0]
+    dataframe['best_ask'] = ob['asks'][0][0]
-        dataframe['best_ask'] = ob['asks'][0][0]
 ```
 
 The orderbook structure is aligned with the order structure from [ccxt](https://github.com/ccxt/ccxt/wiki/Manual#order-book-structure), so the result will look as follows:
@@ -717,12 +713,11 @@ Therefore, using `ob['bids'][0][0]` as demonstrated above will result in using t
 ### *ticker(pair)*
 
 ``` python
-if self.dp:
+if self.dp.runmode.value in ('live', 'dry_run'):
-    if self.dp.runmode.value in ('live', 'dry_run'):
+    ticker = self.dp.ticker(metadata['pair'])
-        ticker = self.dp.ticker(metadata['pair'])
+    dataframe['last_price'] = ticker['last']
-        dataframe['last_price'] = ticker['last']
+    dataframe['volume24h'] = ticker['quoteVolume']
-        dataframe['volume24h'] = ticker['quoteVolume']
+    dataframe['vwap'] = ticker['vwap']
-        dataframe['vwap'] = ticker['vwap']
 ```
 
 !!! Warning
@@ -732,7 +727,7 @@ if self.dp:
     data returned from the exchange and add appropriate error handling / defaults.
 
 !!! Warning "Warning about backtesting"
-    This method will always return up-to-date values - so usage during backtesting / hyperopt will lead to wrong results.
+    This method will always return up-to-date values - so usage during backtesting / hyperopt without runmode checks will lead to wrong results.
 
 ### Send Notification
 

freqtrade/exchange/__init__.py

@@ -9,12 +9,13 @@ from freqtrade.exchange.bitpanda import Bitpanda
 from freqtrade.exchange.bittrex import Bittrex
 from freqtrade.exchange.bybit import Bybit
 from freqtrade.exchange.coinbasepro import Coinbasepro
-from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
+from freqtrade.exchange.exchange import (amount_to_precision, available_exchanges, ccxt_exchanges,
-                                         is_exchange_known_ccxt, is_exchange_officially_supported,
+                                         date_minus_candles, is_exchange_known_ccxt,
-                                         market_is_active, timeframe_to_minutes, timeframe_to_msecs,
+                                         is_exchange_officially_supported, market_is_active,
-                                         timeframe_to_next_date, timeframe_to_prev_date,
+                                         price_to_precision, timeframe_to_minutes,
-                                         timeframe_to_seconds, validate_exchange,
+                                         timeframe_to_msecs, timeframe_to_next_date,
-                                         validate_exchanges)
+                                         timeframe_to_prev_date, timeframe_to_seconds,
+                                         validate_exchange, validate_exchanges)
 from freqtrade.exchange.ftx import Ftx
 from freqtrade.exchange.gateio import Gateio
 from freqtrade.exchange.hitbtc import Hitbtc

freqtrade/exchange/exchange.py

@@ -116,6 +116,7 @@ class Exchange:
         self._last_markets_refresh: int = 0
 
         # Cache for 10 minutes ...
+        self._cache_lock = Lock()
         self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=2, ttl=60 * 10)
         # Cache values for 1800 to avoid frequent polling of the exchange for prices
         # Caching only applies to RPC methods, so prices for open trades are still
@@ -680,45 +681,35 @@ class Exchange:
         """
         return endpoint in self._api.has and self._api.has[endpoint]
 
+    def get_precision_amount(self, pair: str) -> Optional[float]:
+        """
+        Returns the amount precision of the exchange.
+        :param pair: Pair to get precision for
+        :return: precision for amount or None. Must be used in combination with precisionMode
+        """
+        return self.markets.get(pair, {}).get('precision', {}).get('amount', None)
+
+    def get_precision_price(self, pair: str) -> Optional[float]:
+        """
+        Returns the price precision of the exchange.
+        :param pair: Pair to get precision for
+        :return: precision for price or None. Must be used in combination with precisionMode
+        """
+        return self.markets.get(pair, {}).get('precision', {}).get('price', None)
+
     def amount_to_precision(self, pair: str, amount: float) -> float:
         """
         Returns the amount to buy or sell to a precision the Exchange accepts
-        Re-implementation of ccxt internal methods - ensuring we can test the result is correct
-        based on our definitions.
-        """
-        if self.markets[pair]['precision']['amount'] is not None:
-            amount = float(decimal_to_precision(amount, rounding_mode=TRUNCATE,
-                                                precision=self.markets[pair]['precision']['amount'],
-                                                counting_mode=self.precisionMode,
-                                                ))
 
-        return amount
+        """
+        return amount_to_precision(amount, self.get_precision_amount(pair), self.precisionMode)
 
     def price_to_precision(self, pair: str, price: float) -> float:
         """
         Returns the price rounded up to the precision the Exchange accepts.
-        Partial Re-implementation of ccxt internal method decimal_to_precision(),
-        which does not support rounding up
-        TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and
-        align with amount_to_precision().
         Rounds up
         """
-        if self.markets[pair]['precision']['price']:
+        return price_to_precision(price, self.get_precision_price(pair), self.precisionMode)
-            # price = float(decimal_to_precision(price, rounding_mode=ROUND,
-            #                                    precision=self.markets[pair]['precision']['price'],
-            #                                    counting_mode=self.precisionMode,
-            #                                    ))
-            if self.precisionMode == TICK_SIZE:
-                precision = FtPrecise(self.markets[pair]['precision']['price'])
-                price_str = FtPrecise(price)
-                missing = price_str % precision
-                if not missing == FtPrecise("0"):
-                    price = round(float(str(price_str - missing + precision)), 14)
-            else:
-                symbol_prec = self.markets[pair]['precision']['price']
-                big_price = price * pow(10, symbol_prec)
-                price = ceil(big_price) / pow(10, symbol_prec)
-        return price
 
     def price_get_one_pip(self, pair: str, price: float) -> float:
         """
@@ -1019,7 +1010,8 @@ class Exchange:
         time_in_force: str = 'gtc',
     ) -> Dict:
         if self._config['dry_run']:
-            dry_order = self.create_dry_run_order(pair, ordertype, side, amount, rate, leverage)
+            dry_order = self.create_dry_run_order(
+                pair, ordertype, side, amount, self.price_to_precision(pair, rate), leverage)
             return dry_order
 
         params = self._get_params(side, ordertype, leverage, reduceOnly, time_in_force)
@@ -1387,12 +1379,14 @@ class Exchange:
         if not self.exchange_has('fetchBidsAsks'):
             return {}
         if cached:
-            tickers = self._fetch_tickers_cache.get('fetch_bids_asks')
+            with self._cache_lock:
+                tickers = self._fetch_tickers_cache.get('fetch_bids_asks')
             if tickers:
                 return tickers
         try:
             tickers = self._api.fetch_bids_asks(symbols)
-            self._fetch_tickers_cache['fetch_bids_asks'] = tickers
+            with self._cache_lock:
+                self._fetch_tickers_cache['fetch_bids_asks'] = tickers
             return tickers
         except ccxt.NotSupported as e:
             raise OperationalException(
@@ -1413,12 +1407,14 @@ class Exchange:
         :return: fetch_tickers result
         """
         if cached:
-            tickers = self._fetch_tickers_cache.get('fetch_tickers')
+            with self._cache_lock:
+                tickers = self._fetch_tickers_cache.get('fetch_tickers')
             if tickers:
                 return tickers
         try:
             tickers = self._api.fetch_tickers(symbols)
-            self._fetch_tickers_cache['fetch_tickers'] = tickers
+            with self._cache_lock:
+                self._fetch_tickers_cache['fetch_tickers'] = tickers
             return tickers
         except ccxt.NotSupported as e:
             raise OperationalException(
@@ -1527,7 +1523,8 @@ class Exchange:
 
         cache_rate: TTLCache = self._entry_rate_cache if side == "entry" else self._exit_rate_cache
         if not refresh:
-            rate = cache_rate.get(pair)
+            with self._cache_lock:
+                rate = cache_rate.get(pair)
             # Check if cache has been invalidated
             if rate:
                 logger.debug(f"Using cached {side} rate for {pair}.")
@@ -1572,7 +1569,8 @@ class Exchange:
 
         if rate is None:
             raise PricingError(f"{name}-Rate for {pair} was empty.")
-        cache_rate[pair] = rate
+        with self._cache_lock:
+            cache_rate[pair] = rate
 
         return rate
 
@@ -1580,8 +1578,9 @@ class Exchange:
         entry_rate = None
         exit_rate = None
         if not refresh:
-            entry_rate = self._entry_rate_cache.get(pair)
+            with self._cache_lock:
-            exit_rate = self._exit_rate_cache.get(pair)
+                entry_rate = self._entry_rate_cache.get(pair)
+                exit_rate = self._exit_rate_cache.get(pair)
             if entry_rate:
                 logger.debug(f"Using cached buy rate for {pair}.")
             if exit_rate:
@@ -2246,10 +2245,14 @@ class Exchange:
 
                 coros = [self.get_market_leverage_tiers(symbol) for symbol in sorted(symbols)]
 
+                async def gather_results():
+                    return await asyncio.gather(*input_coro, return_exceptions=True)
+
                 for input_coro in chunks(coros, 100):
 
-                    results = self.loop.run_until_complete(
+                    with self._loop_lock:
-                        asyncio.gather(*input_coro, return_exceptions=True))
+                        results = self.loop.run_until_complete(gather_results())
+
                     for symbol, res in results:
                         tiers[symbol] = res
 
@@ -2849,3 +2852,61 @@ def market_is_active(market: Dict) -> bool:
     # See https://github.com/ccxt/ccxt/issues/4874,
     # https://github.com/ccxt/ccxt/issues/4075#issuecomment-434760520
     return market.get('active', True) is not False
+
+
+def amount_to_precision(amount: float, amount_precision: Optional[float],
+                        precisionMode: Optional[int]) -> float:
+    """
+    Returns the amount to buy or sell to a precision the Exchange accepts
+    Re-implementation of ccxt internal methods - ensuring we can test the result is correct
+    based on our definitions.
+    :param amount: amount to truncate
+    :param amount_precision: amount precision to use.
+                             should be retrieved from markets[pair]['precision']['amount']
+    :param precisionMode: precision mode to use. Should be used from precisionMode
+                          one of ccxt's DECIMAL_PLACES, SIGNIFICANT_DIGITS, or TICK_SIZE
+    :return: truncated amount
+    """
+    if amount_precision is not None and precisionMode is not None:
+        precision = int(amount_precision) if precisionMode != TICK_SIZE else amount_precision
+        # precision must be an int for non-ticksize inputs.
+        amount = float(decimal_to_precision(amount, rounding_mode=TRUNCATE,
+                                            precision=precision,
+                                            counting_mode=precisionMode,
+                                            ))
+
+    return amount
+
+
+def price_to_precision(price: float, price_precision: Optional[float],
+                       precisionMode: Optional[int]) -> float:
+    """
+    Returns the price rounded up to the precision the Exchange accepts.
+    Partial Re-implementation of ccxt internal method decimal_to_precision(),
+    which does not support rounding up
+    TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and
+    align with amount_to_precision().
+    !!! Rounds up
+    :param price: price to convert
+    :param price_precision: price precision to use. Used from markets[pair]['precision']['price']
+    :param precisionMode: precision mode to use. Should be used from precisionMode
+                          one of ccxt's DECIMAL_PLACES, SIGNIFICANT_DIGITS, or TICK_SIZE
+    :return: price rounded up to the precision the Exchange accepts
+
+    """
+    if price_precision is not None and precisionMode is not None:
+        # price = float(decimal_to_precision(price, rounding_mode=ROUND,
+        #                                    precision=price_precision,
+        #                                    counting_mode=self.precisionMode,
+        #                                    ))
+        if precisionMode == TICK_SIZE:
+            precision = FtPrecise(price_precision)
+            price_str = FtPrecise(price)
+            missing = price_str % precision
+            if not missing == FtPrecise("0"):
+                price = round(float(str(price_str - missing + precision)), 14)
+        else:
+            symbol_prec = price_precision
+            big_price = price * pow(10, symbol_prec)
+            price = ceil(big_price) / pow(10, symbol_prec)
+    return price

freqtrade/exchange/okx.py

@@ -7,9 +7,8 @@ from freqtrade.constants import BuySell
 from freqtrade.enums import MarginMode, TradingMode
 from freqtrade.enums.candletype import CandleType
 from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError
-from freqtrade.exchange import Exchange
+from freqtrade.exchange import Exchange, date_minus_candles
 from freqtrade.exchange.common import retrier
-from freqtrade.exchange.exchange import date_minus_candles
 
 
 logger = logging.getLogger(__name__)

freqtrade/freqai/prediction_models/LightGBMClassifier.py

@@ -26,13 +26,18 @@ class LightGBMClassifier(BaseClassifierModel):
 
         if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0:
             eval_set = None
+            test_weights = None
         else:
-            eval_set = (data_dictionary["test_features"], data_dictionary["test_labels"])
+            eval_set = (data_dictionary["test_features"].to_numpy(),
-        X = data_dictionary["train_features"]
+                        data_dictionary["test_labels"].to_numpy()[:, 0])
-        y = data_dictionary["train_labels"]
+            test_weights = data_dictionary["test_weights"]
+        X = data_dictionary["train_features"].to_numpy()
+        y = data_dictionary["train_labels"].to_numpy()[:, 0]
+        train_weights = data_dictionary["train_weights"]
 
         model = LGBMClassifier(**self.model_training_parameters)
 
-        model.fit(X=X, y=y, eval_set=eval_set)
+        model.fit(X=X, y=y, eval_set=eval_set, sample_weight=train_weights,
+                  eval_sample_weight=[test_weights])
 
         return model

freqtrade/freqai/prediction_models/LightGBMRegressor.py

@@ -27,13 +27,17 @@ class LightGBMRegressor(BaseRegressionModel):
 
         if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0:
             eval_set = None
+            eval_weights = None
         else:
             eval_set = (data_dictionary["test_features"], data_dictionary["test_labels"])
+            eval_weights = data_dictionary["test_weights"]
         X = data_dictionary["train_features"]
         y = data_dictionary["train_labels"]
+        train_weights = data_dictionary["train_weights"]
 
         model = LGBMRegressor(**self.model_training_parameters)
 
-        model.fit(X=X, y=y, eval_set=eval_set)
+        model.fit(X=X, y=y, eval_set=eval_set, sample_weight=train_weights,
+                  eval_sample_weight=[eval_weights])
 
         return model

freqtrade/freqtradebot.py

@@ -5,7 +5,6 @@ import copy
 import logging
 import traceback
 from datetime import datetime, time, timedelta, timezone
-from decimal import Decimal
 from math import isclose
 from threading import Lock
 from typing import Any, Dict, List, Optional, Tuple
@@ -33,6 +32,7 @@ from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.rpc import RPCManager
 from freqtrade.strategy.interface import IStrategy
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
+from freqtrade.util import FtPrecise
 from freqtrade.wallets import Wallets
 
 
@@ -159,6 +159,8 @@ class FreqtradeBot(LoggingMixin):
         performs startup tasks
         """
         self.rpc.startup_messages(self.config, self.pairlists, self.protections)
+        # Update older trades with precision and precision mode
+        self.startup_backpopulate_precision()
         if not self.edge:
             # Adjust stoploss if it was changed
             Trade.stoploss_reinitialization(self.strategy.stoploss)
@@ -286,6 +288,17 @@ class FreqtradeBot(LoggingMixin):
         else:
             return 0.0
 
+    def startup_backpopulate_precision(self):
+
+        trades = Trade.get_trades([Trade.precision_mode.is_(None)])
+        for trade in trades:
+            if trade.exchange != self.exchange.id:
+                continue
+            trade.precision_mode = self.exchange.precisionMode
+            trade.amount_precision = self.exchange.get_precision_amount(trade.pair)
+            trade.price_precision = self.exchange.get_precision_price(trade.pair)
+        Trade.commit()
+
     def startup_update_open_orders(self):
         """
         Updates open orders based on order list kept in the database.
@@ -565,7 +578,7 @@ class FreqtradeBot(LoggingMixin):
 
         if stake_amount is not None and stake_amount < 0.0:
             # We should decrease our position
-            amount = abs(float(Decimal(stake_amount) / Decimal(current_exit_rate)))
+            amount = abs(float(FtPrecise(stake_amount) / FtPrecise(current_exit_rate)))
             if amount > trade.amount:
                 # This is currently ineffective as remaining would become < min tradable
                 # Fixing this would require checking for 0.0 there -
@@ -738,7 +751,10 @@ class FreqtradeBot(LoggingMixin):
                 leverage=leverage,
                 is_short=is_short,
                 trading_mode=self.trading_mode,
-                funding_fees=funding_fees
+                funding_fees=funding_fees,
+                amount_precision=self.exchange.get_precision_amount(pair),
+                price_precision=self.exchange.get_precision_price(pair),
+                precision_mode=self.exchange.precisionMode,
             )
         else:
             # This is additional buy, we reset fee_open_currency so timeout checking can work
@@ -1866,6 +1882,9 @@ class FreqtradeBot(LoggingMixin):
             if fee_rate is not None and fee_rate < 0.02:
                 # Only update if fee-rate is < 2%
                 trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', ''))
+            else:
+                logger.warning(
+                    f"Not updating {order.get('side', '')}-fee - rate: {fee_rate}, {fee_currency}.")
 
         if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC):
             # * Leverage could be a cause for this warning

freqtrade/leverage/interest.py

@@ -1,20 +1,20 @@
-from decimal import Decimal
 from math import ceil
 
 from freqtrade.exceptions import OperationalException
+from freqtrade.util import FtPrecise
 
 
-one = Decimal(1.0)
+one = FtPrecise(1.0)
-four = Decimal(4.0)
+four = FtPrecise(4.0)
-twenty_four = Decimal(24.0)
+twenty_four = FtPrecise(24.0)
 
 
 def interest(
     exchange_name: str,
-    borrowed: Decimal,
+    borrowed: FtPrecise,
-    rate: Decimal,
+    rate: FtPrecise,
-    hours: Decimal
+    hours: FtPrecise
-) -> Decimal:
+) -> FtPrecise:
     """
     Equation to calculate interest on margin trades
 
@@ -31,13 +31,13 @@ def interest(
     """
     exchange_name = exchange_name.lower()
     if exchange_name == "binance":
-        return borrowed * rate * ceil(hours) / twenty_four
+        return borrowed * rate * FtPrecise(ceil(hours)) / twenty_four
     elif exchange_name == "kraken":
         # Rounded based on https://kraken-fees-calculator.github.io/
-        return borrowed * rate * (one + ceil(hours / four))
+        return borrowed * rate * (one + FtPrecise(ceil(hours / four)))
     elif exchange_name == "ftx":
         # As Explained under #Interest rates section in
         # https://help.ftx.com/hc/en-us/articles/360053007671-Spot-Margin-Trading-Explainer
-        return borrowed * rate * ceil(hours) / twenty_four
+        return borrowed * rate * FtPrecise(ceil(hours)) / twenty_four
     else:
         raise OperationalException(f"Leverage not available on {exchange_name} with freqtrade")

freqtrade/optimize/backtesting.py

@@ -131,6 +131,7 @@ class Backtesting:
             self.fee = config['fee']
         else:
             self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0])
+        self.precision_mode = self.exchange.precisionMode
 
         self.timerange = TimeRange.parse_timerange(
             None if self.config.get('timerange') is None else str(self.config.get('timerange')))
@@ -849,6 +850,9 @@ class Backtesting:
                     trading_mode=self.trading_mode,
                     leverage=leverage,
                     # interest_rate=interest_rate,
+                    amount_precision=self.exchange.get_precision_amount(pair),
+                    price_precision=self.exchange.get_precision_price(pair),
+                    precision_mode=self.precision_mode,
                     orders=[],
                 )
 

freqtrade/optimize/hyperopt.py

@@ -483,6 +483,7 @@ class Hyperopt:
         self.backtesting.exchange._api_async = None
         self.backtesting.exchange.loop = None  # type: ignore
         self.backtesting.exchange._loop_lock = None  # type: ignore
+        self.backtesting.exchange._cache_lock = None  # type: ignore
         # self.backtesting.exchange = None  # type: ignore
         self.backtesting.pairlists = None  # type: ignore
 

freqtrade/persistence/migrations.py

@@ -130,6 +130,10 @@ def migrate_trades_and_orders_table(
                                        get_column_def(cols, 'sell_order_status', 'null'))
     amount_requested = get_column_def(cols, 'amount_requested', 'amount')
 
+    amount_precision = get_column_def(cols, 'amount_precision', 'null')
+    price_precision = get_column_def(cols, 'price_precision', 'null')
+    precision_mode = get_column_def(cols, 'precision_mode', 'null')
+
     # Schema migration necessary
     with engine.begin() as connection:
         connection.execute(text(f"alter table trades rename to {trade_back_name}"))
@@ -156,7 +160,8 @@ def migrate_trades_and_orders_table(
             max_rate, min_rate, exit_reason, exit_order_status, strategy, enter_tag,
             timeframe, open_trade_value, close_profit_abs,
             trading_mode, leverage, liquidation_price, is_short,
-            interest_rate, funding_fees, realized_profit
+            interest_rate, funding_fees, realized_profit,
+            amount_precision, price_precision, precision_mode
             )
         select id, lower(exchange), pair, {base_currency} base_currency,
             {stake_currency} stake_currency,
@@ -182,7 +187,9 @@ def migrate_trades_and_orders_table(
             {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs,
             {trading_mode} trading_mode, {leverage} leverage, {liquidation_price} liquidation_price,
             {is_short} is_short, {interest_rate} interest_rate,
-            {funding_fees} funding_fees, {realized_profit} realized_profit
+            {funding_fees} funding_fees, {realized_profit} realized_profit,
+            {amount_precision} amount_precision, {price_precision} price_precision,
+            {precision_mode} precision_mode
             from {trade_back_name}
             """))
 
@@ -300,7 +307,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
     # Migrates both trades and orders table!
     # if ('orders' not in previous_tables
     # or not has_column(cols_orders, 'stop_price')):
-    if not has_column(cols_trades, 'realized_profit'):
+    if not has_column(cols_trades, 'precision_mode'):
         logger.info(f"Running database migration for trades - "
                     f"backup: {table_back_name}, {order_table_bak_name}")
         migrate_trades_and_orders_table(

freqtrade/persistence/trade_model.py

@@ -3,7 +3,6 @@ This module contains the class to persist trades into SQLite
 """
 import logging
 from datetime import datetime, timedelta, timezone
-from decimal import Decimal
 from math import isclose
 from typing import Any, Dict, List, Optional
 
@@ -15,8 +14,10 @@ from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPE
                                  BuySell, LongShort)
 from freqtrade.enums import ExitType, TradingMode
 from freqtrade.exceptions import DependencyException, OperationalException
+from freqtrade.exchange import amount_to_precision, price_to_precision
 from freqtrade.leverage import interest
 from freqtrade.persistence.base import _DECL_BASE
+from freqtrade.util import FtPrecise
 
 
 logger = logging.getLogger(__name__)
@@ -292,6 +293,9 @@ class LocalTrade():
     timeframe: Optional[int] = None
 
     trading_mode: TradingMode = TradingMode.SPOT
+    amount_precision: Optional[float] = None
+    price_precision: Optional[float] = None
+    precision_mode: Optional[int] = None
 
     # Leverage trading properties
     liquidation_price: Optional[float] = None
@@ -523,9 +527,10 @@ class LocalTrade():
         """
         Method used internally to set self.stop_loss.
         """
+        stop_loss_norm = price_to_precision(stop_loss, self.price_precision, self.precision_mode)
         if not self.stop_loss:
-            self.initial_stop_loss = stop_loss
+            self.initial_stop_loss = stop_loss_norm
-        self.stop_loss = stop_loss
+        self.stop_loss = stop_loss_norm
 
         self.stop_loss_pct = -1 * abs(percent)
         self.stoploss_last_update = datetime.utcnow()
@@ -553,7 +558,8 @@ class LocalTrade():
         # no stop loss assigned yet
         if self.initial_stop_loss_pct is None or refresh:
             self.__set_stop_loss(new_loss, stoploss)
-            self.initial_stop_loss = new_loss
+            self.initial_stop_loss = price_to_precision(
+                new_loss, self.price_precision, self.precision_mode)
             self.initial_stop_loss_pct = -1 * abs(stoploss)
 
         # evaluate if the stop loss needs to be updated
@@ -617,7 +623,8 @@ class LocalTrade():
             else:
                 logger.warning(
                     f'Got different open_order_id {self.open_order_id} != {order.order_id}')
-            if isclose(order.safe_amount_after_fee, self.amount, abs_tol=MATH_CLOSE_PREC):
+            amount_tr = amount_to_precision(self.amount, self.amount_precision, self.precision_mode)
+            if isclose(order.safe_amount_after_fee, amount_tr, abs_tol=MATH_CLOSE_PREC):
                 self.close(order.safe_price)
             else:
                 self.recalc_trade_from_orders()
@@ -694,8 +701,8 @@ class LocalTrade():
         Calculate the open_rate including open_fee.
         :return: Price in of the open trade incl. Fees
         """
-        open_trade = Decimal(amount) * Decimal(open_rate)
+        open_trade = FtPrecise(amount) * FtPrecise(open_rate)
-        fees = open_trade * Decimal(self.fee_open)
+        fees = open_trade * FtPrecise(self.fee_open)
         if self.is_short:
             return float(open_trade - fees)
         else:
@@ -708,30 +715,30 @@ class LocalTrade():
         """
         self.open_trade_value = self._calc_open_trade_value(self.amount, self.open_rate)
 
-    def calculate_interest(self) -> Decimal:
+    def calculate_interest(self) -> FtPrecise:
         """
         Calculate interest for this trade. Only applicable for Margin trading.
         """
-        zero = Decimal(0.0)
+        zero = FtPrecise(0.0)
         # If nothing was borrowed
         if self.trading_mode != TradingMode.MARGIN or self.has_no_leverage:
             return zero
 
         open_date = self.open_date.replace(tzinfo=None)
         now = (self.close_date or datetime.now(timezone.utc)).replace(tzinfo=None)
-        sec_per_hour = Decimal(3600)
+        sec_per_hour = FtPrecise(3600)
-        total_seconds = Decimal((now - open_date).total_seconds())
+        total_seconds = FtPrecise((now - open_date).total_seconds())
         hours = total_seconds / sec_per_hour or zero
 
-        rate = Decimal(self.interest_rate)
+        rate = FtPrecise(self.interest_rate)
-        borrowed = Decimal(self.borrowed)
+        borrowed = FtPrecise(self.borrowed)
 
         return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours)
 
-    def _calc_base_close(self, amount: Decimal, rate: float, fee: float) -> Decimal:
+    def _calc_base_close(self, amount: FtPrecise, rate: float, fee: float) -> FtPrecise:
 
-        close_trade = amount * Decimal(rate)
+        close_trade = amount * FtPrecise(rate)
-        fees = close_trade * Decimal(fee)
+        fees = close_trade * FtPrecise(fee)
 
         if self.is_short:
             return close_trade + fees
@@ -747,7 +754,7 @@ class LocalTrade():
         if rate is None and not self.close_rate:
             return 0.0
 
-        amount1 = Decimal(amount or self.amount)
+        amount1 = FtPrecise(amount or self.amount)
         trading_mode = self.trading_mode or TradingMode.SPOT
 
         if trading_mode == TradingMode.SPOT:
@@ -826,12 +833,12 @@ class LocalTrade():
 
         return float(f"{profit_ratio:.8f}")
 
-    def recalc_trade_from_orders(self, is_closing: bool = False):
+    def recalc_trade_from_orders(self, *, is_closing: bool = False):
-
+        ZERO = FtPrecise(0.0)
-        current_amount = 0.0
+        current_amount = FtPrecise(0.0)
-        current_stake = 0.0
+        current_stake = FtPrecise(0.0)
         total_stake = 0.0  # Total stake after all buy orders (does not subtract!)
-        avg_price = 0.0
+        avg_price = FtPrecise(0.0)
         close_profit = 0.0
         close_profit_abs = 0.0
 
@@ -839,28 +846,29 @@ class LocalTrade():
             if o.ft_is_open or not o.filled:
                 continue
 
-            tmp_amount = o.safe_amount_after_fee
+            tmp_amount = FtPrecise(o.safe_amount_after_fee)
-            tmp_price = o.safe_price
+            tmp_price = FtPrecise(o.safe_price)
 
             is_exit = o.ft_order_side != self.entry_side
-            side = -1 if is_exit else 1
+            side = FtPrecise(-1 if is_exit else 1)
-            if tmp_amount > 0.0 and tmp_price is not None:
+            if tmp_amount > ZERO and tmp_price is not None:
                 current_amount += tmp_amount * side
                 price = avg_price if is_exit else tmp_price
                 current_stake += price * tmp_amount * side
 
-                if current_amount > 0:
+                if current_amount > ZERO:
                     avg_price = current_stake / current_amount
 
             if is_exit:
                 # Process partial exits
                 exit_rate = o.safe_price
                 exit_amount = o.safe_amount_after_fee
-                profit = self.calc_profit(rate=exit_rate, amount=exit_amount, open_rate=avg_price)
+                profit = self.calc_profit(rate=exit_rate, amount=exit_amount,
+                                          open_rate=float(avg_price))
                 close_profit_abs += profit
                 close_profit = self.calc_profit_ratio(
                     exit_rate, amount=exit_amount, open_rate=avg_price)
-                if current_amount <= 0:
+                if current_amount <= ZERO:
                     profit = close_profit_abs
             else:
                 total_stake = total_stake + self._calc_open_trade_value(tmp_amount, price)
@@ -870,13 +878,15 @@ class LocalTrade():
             self.realized_profit = close_profit_abs
             self.close_profit_abs = profit
 
-        if current_amount > 0:
+        current_amount_tr = amount_to_precision(float(current_amount),
+                                                self.amount_precision, self.precision_mode)
+        if current_amount_tr > 0.0:
             # Trade is still open
             # Leverage not updated, as we don't allow changing leverage through DCA at the moment.
-            self.open_rate = current_stake / current_amount
+            self.open_rate = float(current_stake / current_amount)
-            self.stake_amount = current_stake / (self.leverage or 1.0)
+            self.amount = current_amount_tr
-            self.amount = current_amount
+            self.stake_amount = float(current_stake) / (self.leverage or 1.0)
-            self.fee_open_cost = self.fee_open * current_stake
+            self.fee_open_cost = self.fee_open * float(current_stake)
             self.recalc_open_trade_value()
             if self.stop_loss_pct is not None and self.open_rate is not None:
                 self.adjust_stop_loss(self.open_rate, self.stop_loss_pct)
@@ -1119,6 +1129,9 @@ class Trade(_DECL_BASE, LocalTrade):
     timeframe = Column(Integer, nullable=True)
 
     trading_mode = Column(Enum(TradingMode), nullable=True)
+    amount_precision = Column(Float, nullable=True)
+    price_precision = Column(Float, nullable=True)
+    precision_mode = Column(Integer, nullable=True)
 
     # Leverage trading properties
     leverage = Column(Float, nullable=True, default=1.0)

freqtrade/strategy/interface.py

@@ -617,9 +617,6 @@ class IStrategy(ABC, HyperStrategyMixin):
                 )
                 informative_pairs.append(pair_tf)
             else:
-                if not self.dp:
-                    raise OperationalException('@informative decorator with unspecified asset '
-                                               'requires DataProvider instance.')
                 for pair in self.dp.current_whitelist():
                     informative_pairs.append((pair, inf_data.timeframe, candle_type))
         return list(set(informative_pairs))
@@ -713,10 +710,9 @@ class IStrategy(ABC, HyperStrategyMixin):
             # Defs that only make change on new candle data.
             dataframe = self.analyze_ticker(dataframe, metadata)
             self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date']
-            if self.dp:
+            self.dp._set_cached_df(
-                self.dp._set_cached_df(
+                pair, self.timeframe, dataframe,
-                    pair, self.timeframe, dataframe,
+                candle_type=self.config.get('candle_type_def', CandleType.SPOT))
-                    candle_type=self.config.get('candle_type_def', CandleType.SPOT))
         else:
             logger.debug("Skipping TA Analysis for already analyzed candle")
             dataframe[SignalType.ENTER_LONG.value] = 0
@@ -737,8 +733,6 @@ class IStrategy(ABC, HyperStrategyMixin):
         The analyzed dataframe is then accessible via `dp.get_analyzed_dataframe()`.
         :param pair: Pair to analyze.
         """
-        if not self.dp:
-            raise OperationalException("DataProvider not found.")
         dataframe = self.dp.ohlcv(
             pair, self.timeframe, candle_type=self.config.get('candle_type_def', CandleType.SPOT)
         )

freqtrade/templates/subtemplates/exchange_gateio.j2

@@ -0,0 +1,12 @@
+"exchange": {
+    "name": "{{ exchange_name | lower }}",
+    "key": "{{ exchange_key }}",
+    "secret": "{{ exchange_secret }}",
+    "unknown_fee_rate": 1,
+    "ccxt_config": {},
+    "ccxt_async_config": {},
+    "pair_whitelist": [
+    ],
+    "pair_blacklist": [
+    ]
+}

freqtrade/templates/subtemplates/exchange_generic.j2

@@ -5,9 +5,7 @@
     "ccxt_config": {},
     "ccxt_async_config": {},
     "pair_whitelist": [
-
     ],
     "pair_blacklist": [
-
     ]
 }

setup.py

@@ -49,7 +49,7 @@ setup(
     ],
     install_requires=[
         # from requirements.txt
-        'ccxt>=1.83.12',
+        'ccxt>=1.92.9',
         'SQLAlchemy',
         'python-telegram-bot>=13.4',
         'arrow>=0.17.0',

tests/conftest_trades_usdt.py

@@ -81,7 +81,7 @@ def mock_trade_usdt_1(fee, is_short: bool):
 def mock_order_usdt_2(is_short: bool):
     return {
         'id': f'1235_{direc(is_short)}',
-        'symbol': 'ETC/USDT',
+        'symbol': 'NEO/USDT',
         'status': 'closed',
         'side': entry_side(is_short),
         'type': 'limit',
@@ -95,7 +95,7 @@ def mock_order_usdt_2(is_short: bool):
 def mock_order_usdt_2_exit(is_short: bool):
     return {
         'id': f'12366_{direc(is_short)}',
-        'symbol': 'ETC/USDT',
+        'symbol': 'NEO/USDT',
         'status': 'closed',
         'side': exit_side(is_short),
         'type': 'limit',
@@ -111,7 +111,7 @@ def mock_trade_usdt_2(fee, is_short: bool):
     Closed trade...
     """
     trade = Trade(
-        pair='ETC/USDT',
+        pair='NEO/USDT',
         stake_amount=200.0,
         amount=100.0,
         amount_requested=100.0,
@@ -132,10 +132,10 @@ def mock_trade_usdt_2(fee, is_short: bool):
         close_date=datetime.now(tz=timezone.utc) - timedelta(minutes=2),
         is_short=is_short,
     )
-    o = Order.parse_from_ccxt_object(mock_order_usdt_2(is_short), 'ETC/USDT', entry_side(is_short))
+    o = Order.parse_from_ccxt_object(mock_order_usdt_2(is_short), 'NEO/USDT', entry_side(is_short))
     trade.orders.append(o)
     o = Order.parse_from_ccxt_object(
-        mock_order_usdt_2_exit(is_short), 'ETC/USDT', exit_side(is_short))
+        mock_order_usdt_2_exit(is_short), 'NEO/USDT', exit_side(is_short))
     trade.orders.append(o)
     return trade
 
@@ -205,7 +205,7 @@ def mock_trade_usdt_3(fee, is_short: bool):
 def mock_order_usdt_4(is_short: bool):
     return {
         'id': f'prod_buy_12345_{direc(is_short)}',
-        'symbol': 'ETC/USDT',
+        'symbol': 'NEO/USDT',
         'status': 'open',
         'side': entry_side(is_short),
         'type': 'limit',
@@ -221,7 +221,7 @@ def mock_trade_usdt_4(fee, is_short: bool):
     Simulate prod entry
     """
     trade = Trade(
-        pair='ETC/USDT',
+        pair='NEO/USDT',
         stake_amount=20.0,
         amount=10.0,
         amount_requested=10.01,
@@ -236,7 +236,7 @@ def mock_trade_usdt_4(fee, is_short: bool):
         timeframe=5,
         is_short=is_short,
     )
-    o = Order.parse_from_ccxt_object(mock_order_usdt_4(is_short), 'ETC/USDT', entry_side(is_short))
+    o = Order.parse_from_ccxt_object(mock_order_usdt_4(is_short), 'NEO/USDT', entry_side(is_short))
     trade.orders.append(o)
     return trade
 

tests/exchange/test_ccxt_precise.py

@@ -78,3 +78,5 @@ def test_FtPrecise():
     assert FtPrecise(-213) == '-213'
     assert str(FtPrecise(-213)) == '-213'
     assert FtPrecise(213.2) == '213.2'
+    assert float(FtPrecise(213.2)) == 213.2
+    assert float(FtPrecise(-213.2)) == -213.2

tests/exchange/test_exchange.py

@@ -14,12 +14,12 @@ from pandas import DataFrame
 from freqtrade.enums import CandleType, MarginMode, TradingMode
 from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOrderException,
                                   OperationalException, PricingError, TemporaryError)
-from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken
+from freqtrade.exchange import (Binance, Bittrex, Exchange, Kraken, amount_to_precision,
+                                date_minus_candles, market_is_active, price_to_precision,
+                                timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date,
+                                timeframe_to_prev_date, timeframe_to_seconds)
 from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT,
                                        calculate_backoff, remove_credentials)
-from freqtrade.exchange.exchange import (date_minus_candles, market_is_active, timeframe_to_minutes,
-                                         timeframe_to_msecs, timeframe_to_next_date,
-                                         timeframe_to_prev_date, timeframe_to_seconds)
 from freqtrade.resolvers.exchange_resolver import ExchangeResolver
 from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re, num_log_has_re
 
@@ -279,62 +279,35 @@ def test_validate_order_time_in_force(default_conf, mocker, caplog):
     ex.validate_order_time_in_force(tif2)
 
 
-@pytest.mark.parametrize("amount,precision_mode,precision,contract_size,expected,trading_mode", [
+@pytest.mark.parametrize("amount,precision_mode,precision,expected", [
-    (2.34559, 2, 4, 1, 2.3455, 'spot'),
+    (2.34559, 2, 4, 2.3455),
-    (2.34559, 2, 5, 1, 2.34559, 'spot'),
+    (2.34559, 2, 5, 2.34559),
-    (2.34559, 2, 3, 1, 2.345, 'spot'),
+    (2.34559, 2, 3, 2.345),
-    (2.9999, 2, 3, 1, 2.999, 'spot'),
+    (2.9999, 2, 3, 2.999),
-    (2.9909, 2, 3, 1, 2.990, 'spot'),
+    (2.9909, 2, 3, 2.990),
-    (2.9909, 2, 0, 1, 2, 'spot'),
+    (2.9909, 2, 0, 2),
-    (29991.5555, 2, 0, 1, 29991, 'spot'),
+    (29991.5555, 2, 0, 29991),
-    (29991.5555, 2, -1, 1, 29990, 'spot'),
+    (29991.5555, 2, -1, 29990),
-    (29991.5555, 2, -2, 1, 29900, 'spot'),
+    (29991.5555, 2, -2, 29900),
     # Tests for Tick-size
-    (2.34559, 4, 0.0001, 1, 2.3455, 'spot'),
+    (2.34559, 4, 0.0001, 2.3455),
-    (2.34559, 4, 0.00001, 1, 2.34559, 'spot'),
+    (2.34559, 4, 0.00001, 2.34559),
-    (2.34559, 4, 0.001, 1, 2.345, 'spot'),
+    (2.34559, 4, 0.001, 2.345),
-    (2.9999, 4, 0.001, 1, 2.999, 'spot'),
+    (2.9999, 4, 0.001, 2.999),
-    (2.9909, 4, 0.001, 1, 2.990, 'spot'),
+    (2.9909, 4, 0.001, 2.990),
-    (2.9909, 4, 0.005, 0.01, 2.99, 'futures'),
+    (2.9909, 4, 0.005, 2.99),
-    (2.9999, 4, 0.005, 10, 2.995, 'futures'),
+    (2.9999, 4, 0.005, 2.995),
 ])
-def test_amount_to_precision(
+def test_amount_to_precision(amount, precision_mode, precision, expected,):
-    default_conf,
-    mocker,
-    amount,
-    precision_mode,
-    precision,
-    contract_size,
-    expected,
-    trading_mode
-):
     """
     Test rounds down
     """
-
-    markets = PropertyMock(return_value={
-        'ETH/BTC': {
-            'contractSize': contract_size,
-            'precision': {
-                'amount': precision
-            }
-        }
-    })
-
-    default_conf['trading_mode'] = trading_mode
-    default_conf['margin_mode'] = 'isolated'
-
-    exchange = get_patched_exchange(mocker, default_conf, id="binance")
     # digits counting mode
     # DECIMAL_PLACES = 2
     # SIGNIFICANT_DIGITS = 3
     # TICK_SIZE = 4
-    mocker.patch('freqtrade.exchange.Exchange.precisionMode',
-                 PropertyMock(return_value=precision_mode))
-    mocker.patch('freqtrade.exchange.Exchange.markets', markets)
 
-    pair = 'ETH/BTC'
+    assert amount_to_precision(amount, precision, precision_mode) == expected
-    assert exchange.amount_to_precision(pair, amount) == expected
 
 
 @pytest.mark.parametrize("price,precision_mode,precision,expected", [
@@ -359,21 +332,13 @@ def test_amount_to_precision(
     (0.000000003483, 4, 1e-12, 0.000000003483),
 
 ])
-def test_price_to_precision(default_conf, mocker, price, precision_mode, precision, expected):
+def test_price_to_precision(price, precision_mode, precision, expected):
-    """Test price to precision"""
-    markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'price': precision}}})
-
-    exchange = get_patched_exchange(mocker, default_conf, id="binance")
-    mocker.patch('freqtrade.exchange.Exchange.markets', markets)
     # digits counting mode
     # DECIMAL_PLACES = 2
     # SIGNIFICANT_DIGITS = 3
     # TICK_SIZE = 4
-    mocker.patch('freqtrade.exchange.Exchange.precisionMode',
-                 PropertyMock(return_value=precision_mode))
 
-    pair = 'ETH/BTC'
+    assert price_to_precision(price, precision, precision_mode) == expected
-    assert exchange.price_to_precision(pair, price) == expected
 
 
 @pytest.mark.parametrize("price,precision_mode,precision,expected", [

tests/leverage/test_interest.py

@@ -1,14 +1,14 @@
-from decimal import Decimal
 from math import isclose
 
 import pytest
 
 from freqtrade.leverage import interest
+from freqtrade.util import FtPrecise
 
 
-ten_mins = Decimal(1 / 6)
+ten_mins = FtPrecise(1 / 6)
-five_hours = Decimal(5.0)
+five_hours = FtPrecise(5.0)
-twentyfive_hours = Decimal(25.0)
+twentyfive_hours = FtPrecise(25.0)
 
 
 @pytest.mark.parametrize('exchange,interest_rate,hours,expected', [
@@ -28,11 +28,11 @@ twentyfive_hours = Decimal(25.0)
     ('ftx', 0.00025, twentyfive_hours, 0.015625),
 ])
 def test_interest(exchange, interest_rate, hours, expected):
-    borrowed = Decimal(60.0)
+    borrowed = FtPrecise(60.0)
 
     assert isclose(interest(
         exchange_name=exchange,
         borrowed=borrowed,
-        rate=Decimal(interest_rate),
+        rate=FtPrecise(interest_rate),
         hours=hours
     ), expected)

tests/plugins/test_pairlist.py

@@ -735,7 +735,7 @@ def test_PerformanceFilter_lookback(mocker, default_conf_usdt, fee, caplog) -> N
     with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
         create_mock_trades_usdt(fee)
         pm.refresh_pairlist()
-        assert pm.whitelist == ['XRP/USDT']
+        assert pm.whitelist == ['XRP/USDT', 'NEO/USDT']
         assert log_has_re(r'Removing pair .* since .* is below .*', caplog)
 
         # Move to "outside" of lookback window, so original sorting is restored.
@@ -762,8 +762,8 @@ def test_PerformanceFilter_keep_mid_order(mocker, default_conf_usdt, fee, caplog
     with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
         create_mock_trades_usdt(fee)
         pm.refresh_pairlist()
-        assert pm.whitelist == ['XRP/USDT', 'ETC/USDT', 'ETH/USDT', 'LTC/USDT',
+        assert pm.whitelist == ['XRP/USDT', 'NEO/USDT', 'ETH/USDT', 'LTC/USDT',
-                                'NEO/USDT', 'TKN/USDT', 'ADA/USDT', ]
+                                'TKN/USDT', 'ADA/USDT', 'ETC/USDT', ]
         # assert log_has_re(r'Removing pair .* since .* is below .*', caplog)
 
         # Move to "outside" of lookback window, so original sorting is restored.

tests/rpc/test_rpc.py

@@ -96,20 +96,20 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'profit_pct': -0.41,
         'profit_abs': -4.09e-06,
         'profit_fiat': ANY,
-        'stop_loss_abs': 9.882e-06,
+        'stop_loss_abs': 9.89e-06,
         'stop_loss_pct': -10.0,
         'stop_loss_ratio': -0.1,
         'stoploss_order_id': None,
         'stoploss_last_update': ANY,
         'stoploss_last_update_timestamp': ANY,
-        'initial_stop_loss_abs': 9.882e-06,
+        'initial_stop_loss_abs': 9.89e-06,
         'initial_stop_loss_pct': -10.0,
         'initial_stop_loss_ratio': -0.1,
-        'stoploss_current_dist': -1.1080000000000002e-06,
+        'stoploss_current_dist': pytest.approx(-1.0999999e-06),
-        'stoploss_current_dist_ratio': -0.10081893,
+        'stoploss_current_dist_ratio': -0.10009099,
-        'stoploss_current_dist_pct': -10.08,
+        'stoploss_current_dist_pct': -10.01,
-        'stoploss_entry_dist': -0.00010475,
+        'stoploss_entry_dist': -0.00010402,
-        'stoploss_entry_dist_ratio': -0.10448878,
+        'stoploss_entry_dist_ratio': -0.10376381,
         'open_order': None,
         'realized_profit': 0.0,
         'exchange': 'binance',
@@ -181,20 +181,20 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         'profit_pct': ANY,
         'profit_abs': ANY,
         'profit_fiat': ANY,
-        'stop_loss_abs': 9.882e-06,
+        'stop_loss_abs': 9.89e-06,
         'stop_loss_pct': -10.0,
         'stop_loss_ratio': -0.1,
         'stoploss_order_id': None,
         'stoploss_last_update': ANY,
         'stoploss_last_update_timestamp': ANY,
-        'initial_stop_loss_abs': 9.882e-06,
+        'initial_stop_loss_abs': 9.89e-06,
         'initial_stop_loss_pct': -10.0,
         'initial_stop_loss_ratio': -0.1,
         'stoploss_current_dist': ANY,
         'stoploss_current_dist_ratio': ANY,
         'stoploss_current_dist_pct': ANY,
-        'stoploss_entry_dist': -0.00010475,
+        'stoploss_entry_dist': -0.00010402,
-        'stoploss_entry_dist_ratio': -0.10448878,
+        'stoploss_entry_dist_ratio': -0.10376381,
         'open_order': None,
         'exchange': 'binance',
         'realized_profit': 0.0,
@@ -761,7 +761,7 @@ def test_rpc_force_exit(default_conf, ticker, fee, mocker) -> None:
     # and trade amount is updated
     rpc._rpc_force_exit('3')
     assert cancel_order_mock.call_count == 1
-    assert trade.amount == filled_amount
+    assert pytest.approx(trade.amount) == filled_amount
 
     mocker.patch(
         'freqtrade.exchange.Exchange.fetch_order',
@@ -830,7 +830,7 @@ def test_performance_handle(default_conf_usdt, ticker, fee, mocker) -> None:
 
     res = rpc._rpc_performance()
     assert len(res) == 3
-    assert res[0]['pair'] == 'ETC/USDT'
+    assert res[0]['pair'] == 'NEO/USDT'
     assert res[0]['count'] == 1
     assert res[0]['profit_pct'] == 5.0
 

tests/rpc/test_rpc_apiserver.py

@@ -892,7 +892,7 @@ def test_api_performance(botclient, fee):
     assert_response(rc)
     assert len(rc.json()) == 2
     assert rc.json() == [{'count': 1, 'pair': 'LTC/ETH', 'profit': 7.61, 'profit_pct': 7.61,
-                          'profit_ratio': 0.07609203, 'profit_abs': 0.01872279},
+                          'profit_ratio': 0.07609203, 'profit_abs': 0.0187228},
                          {'count': 1, 'pair': 'XRP/ETH', 'profit': -5.57, 'profit_pct': -5.57,
                           'profit_ratio': -0.05570419, 'profit_abs': -0.1150375}]
 

tests/test_freqtradebot.py

@@ -4,7 +4,6 @@
 import logging
 import time
 from copy import deepcopy
-from math import isclose
 from typing import List
 from unittest.mock import ANY, MagicMock, PropertyMock, patch
 
@@ -12,7 +11,7 @@ import arrow
 import pytest
 from pandas import DataFrame
 
-from freqtrade.constants import CANCEL_REASON, MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT
+from freqtrade.constants import CANCEL_REASON, UNLIMITED_STAKE_AMOUNT
 from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, RPCMessageType, RunMode,
                              SignalDirection, State)
 from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError,
@@ -23,9 +22,9 @@ from freqtrade.persistence import Order, PairLocks, Trade
 from freqtrade.persistence.models import PairLock
 from freqtrade.plugins.protections.iprotection import ProtectionReturn
 from freqtrade.worker import Worker
-from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker,
+from tests.conftest import (create_mock_trades, create_mock_trades_usdt, get_patched_freqtradebot,
-                            log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal,
+                            get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange,
-                            patch_wallet, patch_whitelist)
+                            patch_get_signal, patch_wallet, patch_whitelist)
 from tests.conftest_trades import (MOCK_TRADE_COUNT, entry_side, exit_side, mock_order_1,
                                    mock_order_2, mock_order_2_sell, mock_order_3, mock_order_3_sell,
                                    mock_order_4, mock_order_5_stoploss, mock_order_6_sell)
@@ -569,7 +568,7 @@ def test_process_trade_creation(default_conf_usdt, ticker_usdt, limit_order, lim
     assert trade.open_date is not None
     assert trade.exchange == 'binance'
     assert trade.open_rate == ticker_usdt.return_value[ticker_side]
-    assert isclose(trade.amount, 60 / ticker_usdt.return_value[ticker_side])
+    assert pytest.approx(trade.amount) == 60 / ticker_usdt.return_value[ticker_side]
 
     assert log_has(
         f'{"Short" if is_short else "Long"} signal found: about create a new trade for ETH/USDT '
@@ -1801,7 +1800,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog,
     # stoploss initially at 20% as edge dictated it.
     assert freqtrade.handle_trade(trade) is False
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
-    assert isclose(trade.stop_loss, 1.76)
+    assert pytest.approx(trade.stop_loss) == 1.76
 
     cancel_order_mock = MagicMock()
     stoploss_order_mock = MagicMock()
@@ -1818,7 +1817,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog,
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
 
     # stoploss should remain the same
-    assert isclose(trade.stop_loss, 1.76)
+    assert pytest.approx(trade.stop_loss) == 1.76
 
     # stoploss on exchange should not be canceled
     cancel_order_mock.assert_not_called()
@@ -2172,7 +2171,7 @@ def test_handle_trade(
 
     assert trade.close_rate == (2.0 if is_short else 2.2)
     assert pytest.approx(trade.close_profit) == close_profit
-    assert trade.calc_profit(trade.close_rate) == 5.685
+    assert pytest.approx(trade.calc_profit(trade.close_rate)) == 5.685
     assert trade.close_date is not None
     assert trade.exit_reason == 'sell_signal1'
 
@@ -4144,6 +4143,7 @@ def test_trailing_stop_loss_positive(
             'last': enter_price + (-0.06 if is_short else 0.06),
         })
     )
+    caplog.clear()
     # stop-loss not reached, adjusted stoploss
     assert freqtrade.handle_trade(trade) is False
     caplog_text = (f"ETH/USDT - Using positive stoploss: 0.01 offset: {offset} profit: "
@@ -4524,11 +4524,8 @@ def test_get_real_amount_wrong_amount_rounding(default_conf_usdt, trades_for_ord
 
     order_obj = Order.parse_from_ccxt_object(buy_order_fee, 'LTC/ETH', 'buy')
     # Amount changes by fee amount.
-    assert isclose(
+    assert pytest.approx(freqtrade.get_real_amount(
-        freqtrade.get_real_amount(trade, limit_buy_order_usdt, order_obj),
+        trade, limit_buy_order_usdt, order_obj)) == amount - (amount * 0.001)
-        amount - (amount * 0.001),
-        abs_tol=MATH_CLOSE_PREC,
-    )
 
 
 def test_get_real_amount_open_trade_usdt(default_conf_usdt, fee, mocker):
@@ -4553,6 +4550,76 @@ def test_get_real_amount_open_trade_usdt(default_conf_usdt, fee, mocker):
     assert freqtrade.get_real_amount(trade, order, order_obj) == amount
 
 
+def test_get_real_amount_in_point(default_conf_usdt, buy_order_fee, fee, mocker, caplog):
+    limit_buy_order_usdt = deepcopy(buy_order_fee)
+
+    # Fees amount in "POINT"
+    trades = [{
+        "info": {
+        },
+        "id": "some_trade_id",
+        "timestamp": 1660092505903,
+        "datetime": "2022-08-10T00:48:25.903Z",
+        "symbol": "CEL/USDT",
+        "order": "some_order_id",
+        "type": None,
+        "side": "sell",
+        "takerOrMaker": "taker",
+        "price": 1.83255,
+        "amount": 83.126,
+        "cost": 152.3325513,
+        "fee": {
+            "currency": "POINT",
+            "cost": 0.3046651026
+        },
+        "fees": [
+            {
+                "cost": "0",
+                "currency": "USDT"
+            },
+            {
+                "cost": "0",
+                "currency": "GT"
+            },
+            {
+                "cost": "0.3046651026",
+                "currency": "POINT"
+            }
+        ]
+    }]
+
+    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades)
+    amount = float(sum(x['amount'] for x in trades))
+    trade = Trade(
+        pair='CEL/USDT',
+        amount=amount,
+        exchange='binance',
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_rate=0.245441,
+        open_order_id="123456"
+    )
+    limit_buy_order_usdt['amount'] = amount
+    freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt)
+
+    order_obj = Order.parse_from_ccxt_object(buy_order_fee, 'LTC/ETH', 'buy')
+    res = freqtrade.get_real_amount(trade, limit_buy_order_usdt, order_obj)
+    assert res == amount
+    assert trade.fee_open_currency is None
+    assert trade.fee_open_cost is None
+    message = "Not updating buy-fee - rate: None, POINT."
+    assert log_has(message, caplog)
+    caplog.clear()
+    freqtrade.config['exchange']['unknown_fee_rate'] = 1
+    res = freqtrade.get_real_amount(trade, limit_buy_order_usdt, order_obj)
+    assert res == amount
+    assert trade.fee_open_currency == 'POINT'
+    assert pytest.approx(trade.fee_open_cost) == 0.3046651026
+    assert trade.fee_open == 0.002
+    assert trade.fee_open != fee.return_value
+    assert not log_has(message, caplog)
+
+
 @pytest.mark.parametrize('amount,fee_abs,wallet,amount_exp', [
     (8.0, 0.0, 10, 8),
     (8.0, 0.0, 0, 8),
@@ -4888,6 +4955,31 @@ def test_startup_update_open_orders(mocker, default_conf_usdt, fee, caplog, is_s
     assert hto_mock.call_args_list[1][0][0]['status'] == 'canceled'
 
 
+@pytest.mark.usefixtures("init_persistence")
+def test_startup_backpopulate_precision(mocker, default_conf_usdt, fee, caplog):
+    freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt)
+    create_mock_trades_usdt(fee)
+
+    trades = Trade.get_trades().all()
+    trades[-1].exchange = 'some_other_exchange'
+    for trade in trades:
+        assert trade.price_precision is None
+        assert trade.amount_precision is None
+        assert trade.precision_mode is None
+
+    freqtrade.startup_backpopulate_precision()
+    trades = Trade.get_trades().all()
+    for trade in trades:
+        if trade.exchange == 'some_other_exchange':
+            assert trade.price_precision is None
+            assert trade.amount_precision is None
+            assert trade.precision_mode is None
+        else:
+            assert trade.price_precision is not None
+            assert trade.amount_precision is not None
+            assert trade.precision_mode is not None
+
+
 @pytest.mark.usefixtures("init_persistence")
 @pytest.mark.parametrize("is_short", [False, True])
 def test_update_closed_trades_without_assigned_fees(mocker, default_conf_usdt, fee, is_short):

tests/test_integration.py

@@ -189,7 +189,7 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, mocker, balance_rati
     assert len(trades) == 5
 
     for trade in trades:
-        assert trade.stake_amount == result1
+        assert pytest.approx(trade.stake_amount) == result1
         # Reset trade open order id's
         trade.open_order_id = None
     trades = Trade.get_open_trades()
@@ -220,8 +220,6 @@ def test_dca_buying(default_conf_usdt, ticker_usdt, fee, mocker) -> None:
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_usdt,
         get_fee=fee,
-        amount_to_precision=lambda s, x, y: y,
-        price_to_precision=lambda s, x, y: y,
     )
 
     patch_get_signal(freqtrade)
@@ -249,7 +247,7 @@ def test_dca_buying(default_conf_usdt, ticker_usdt, fee, mocker) -> None:
     assert len(trade.orders) == 2
     for o in trade.orders:
         assert o.status == "closed"
-    assert trade.stake_amount == 120
+    assert pytest.approx(trade.stake_amount) == 120
 
     # Open-rate averaged between 2.0 and 2.0 * 0.995
     assert trade.open_rate < 2.0
@@ -259,11 +257,11 @@ def test_dca_buying(default_conf_usdt, ticker_usdt, fee, mocker) -> None:
     freqtrade.process()
     trade = Trade.get_trades().first()
     assert len(trade.orders) == 2
-    assert trade.stake_amount == 120
+    assert pytest.approx(trade.stake_amount) == 120
     assert trade.orders[0].amount == 30
-    assert trade.orders[1].amount == 60 / ticker_usdt_modif['bid']
+    assert pytest.approx(trade.orders[1].amount) == 60 / ticker_usdt_modif['bid']
 
-    assert trade.amount == trade.orders[0].amount + trade.orders[1].amount
+    assert pytest.approx(trade.amount) == trade.orders[0].amount + trade.orders[1].amount
     assert trade.nr_of_successful_buys == 2
     assert trade.nr_of_successful_entries == 2
 
@@ -274,7 +272,7 @@ def test_dca_buying(default_conf_usdt, ticker_usdt, fee, mocker) -> None:
     assert trade.is_open is False
     assert trade.orders[0].amount == 30
     assert trade.orders[0].side == 'buy'
-    assert trade.orders[1].amount == 60 / ticker_usdt_modif['bid']
+    assert pytest.approx(trade.orders[1].amount) == 60 / ticker_usdt_modif['bid']
     # Sold everything
     assert trade.orders[-1].side == 'sell'
     assert trade.orders[2].amount == trade.amount

tests/test_persistence.py

@@ -1387,7 +1387,8 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert log_has("trying trades_bak2", caplog)
     assert log_has("Running database migration for trades - backup: trades_bak2, orders_bak0",
                    caplog)
-    assert trade.open_trade_value == trade._calc_open_trade_value(trade.amount, trade.open_rate)
+    assert pytest.approx(trade.open_trade_value) == trade._calc_open_trade_value(
+        trade.amount, trade.open_rate)
     assert trade.close_profit_abs is None
 
     orders = trade.orders