Merge branch 'freqtrade:develop' into feat_bt_cancel_entry_reporting

freqtrade/data/history/hdf5datahandler.py

@@ -40,7 +40,7 @@ class HDF5DataHandler(IDataHandler):
         return [
             (
                 cls.rebuild_pair_from_filename(match[1]),
-                match[2],
+                cls.rebuild_timeframe_from_filename(match[2]),
                 CandleType.from_string(match[3])
             ) for match in _tmp if match and len(match.groups()) > 1]
 
@@ -108,6 +108,10 @@ class HDF5DataHandler(IDataHandler):
             candle_type=candle_type
         )
 
+        if not filename.exists():
+            # Fallback mode for 1M files
+            filename = self._pair_data_filename(
+                self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True)
             if not filename.exists():
                 return pd.DataFrame(columns=self._columns)
         where = []

freqtrade/data/history/idatahandler.py

@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
 
 class IDataHandler(ABC):
 
-    _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=\.)'
+    _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+[a-zA-Z]{1,2})\-?([a-zA-Z_]*)?(?=\.)'
 
     def __init__(self, datadir: Path) -> None:
         self._datadir = datadir
@@ -193,10 +193,14 @@ class IDataHandler(ABC):
         datadir: Path,
         pair: str,
         timeframe: str,
-        candle_type: CandleType
+        candle_type: CandleType,
+        no_timeframe_modify: bool = False
     ) -> Path:
         pair_s = misc.pair_to_filename(pair)
         candle = ""
+        if not no_timeframe_modify:
+            timeframe = cls.timeframe_to_file(timeframe)
+
         if candle_type != CandleType.SPOT:
             datadir = datadir.joinpath('futures')
             candle = f"-{candle_type}"
@@ -210,6 +214,18 @@ class IDataHandler(ABC):
         filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
         return filename
 
+    @staticmethod
+    def timeframe_to_file(timeframe: str):
+        return timeframe.replace('M', 'Mo')
+
+    @staticmethod
+    def rebuild_timeframe_from_filename(timeframe: str) -> str:
+        """
+        converts timeframe from disk to file
+        Replaces mo with M (to avoid problems on case-insensitive filesystems)
+        """
+        return re.sub('1mo', '1M', timeframe, flags=re.IGNORECASE)
+
     @staticmethod
     def rebuild_pair_from_filename(pair: str) -> str:
         """

freqtrade/data/history/jsondatahandler.py

@@ -41,7 +41,7 @@ class JsonDataHandler(IDataHandler):
         return [
             (
                 cls.rebuild_pair_from_filename(match[1]),
-                match[2],
+                cls.rebuild_timeframe_from_filename(match[2]),
                 CandleType.from_string(match[3])
             ) for match in _tmp if match and len(match.groups()) > 1]
 
@@ -103,7 +103,12 @@ class JsonDataHandler(IDataHandler):
         :param candle_type: Any of the enum CandleType (must match trading mode!)
         :return: DataFrame with ohlcv data, or empty DataFrame
         """
-        filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type)
+        filename = self._pair_data_filename(
+            self._datadir, pair, timeframe, candle_type=candle_type)
+        if not filename.exists():
+            # Fallback mode for 1M files
+            filename = self._pair_data_filename(
+                self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True)
             if not filename.exists():
                 return DataFrame(columns=self._columns)
         try:

freqtrade/exchange/exchange.py

@@ -16,8 +16,7 @@ import arrow
 import ccxt
 import ccxt.async_support as ccxt_async
 from cachetools import TTLCache
-from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE,
+from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, Precise, decimal_to_precision
-                                            decimal_to_precision)
 from pandas import DataFrame
 
 from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell,
@@ -704,10 +703,11 @@ class Exchange:
             #                                    counting_mode=self.precisionMode,
             #                                    ))
             if self.precisionMode == TICK_SIZE:
-                precision = self.markets[pair]['precision']['price']
+                precision = Precise(str(self.markets[pair]['precision']['price']))
-                missing = price % precision
+                price_str = Precise(str(price))
-                if missing != 0:
+                missing = price_str % precision
-                    price = round(price - missing + precision, 10)
+                if not missing == Precise("0"):
+                    price = round(float(str(price_str - missing + precision)), 14)
             else:
                 symbol_prec = self.markets[pair]['precision']['price']
                 big_price = price * pow(10, symbol_prec)
@@ -1457,6 +1457,23 @@ class Exchange:
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
+    def _get_price_side(self, side: str, is_short: bool, conf_strategy: Dict) -> str:
+        price_side = conf_strategy['price_side']
+
+        if price_side in ('same', 'other'):
+            price_map = {
+                ('entry', 'long', 'same'): 'bid',
+                ('entry', 'long', 'other'): 'ask',
+                ('entry', 'short', 'same'): 'ask',
+                ('entry', 'short', 'other'): 'bid',
+                ('exit', 'long', 'same'): 'ask',
+                ('exit', 'long', 'other'): 'bid',
+                ('exit', 'short', 'same'): 'bid',
+                ('exit', 'short', 'other'): 'ask',
+            }
+            price_side = price_map[(side, 'short' if is_short else 'long', price_side)]
+        return price_side
+
     def get_rate(self, pair: str, refresh: bool,
                  side: EntryExit, is_short: bool) -> float:
         """
@@ -1483,20 +1500,7 @@ class Exchange:
 
         conf_strategy = self._config.get(strat_name, {})
 
-        price_side = conf_strategy['price_side']
+        price_side = self._get_price_side(side, is_short, conf_strategy)
-
-        if price_side in ('same', 'other'):
-            price_map = {
-                ('entry', 'long', 'same'): 'bid',
-                ('entry', 'long', 'other'): 'ask',
-                ('entry', 'short', 'same'): 'ask',
-                ('entry', 'short', 'other'): 'bid',
-                ('exit', 'long', 'same'): 'ask',
-                ('exit', 'long', 'other'): 'bid',
-                ('exit', 'short', 'same'): 'bid',
-                ('exit', 'short', 'other'): 'ask',
-            }
-            price_side = price_map[(side, 'short' if is_short else 'long', price_side)]
 
         price_side_word = price_side.capitalize()
 

freqtrade/optimize/backtesting.py

@@ -542,11 +542,11 @@ class Backtesting:
 
             trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60)
             try:
-                closerate = self._get_close_rate(row, trade, exit_, trade_dur)
+                close_rate = self._get_close_rate(row, trade, exit_, trade_dur)
             except ValueError:
                 return None
-            # call the custom exit price,with default value as previous closerate
+            # call the custom exit price,with default value as previous close_rate
-            current_profit = trade.calc_profit_ratio(closerate)
+            current_profit = trade.calc_profit_ratio(close_rate)
             order_type = self.strategy.order_types['exit']
             if exit_.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT):
                 # Checks and adds an exit tag, after checking that the length of the
@@ -560,24 +560,24 @@ class Backtesting:
                     exit_reason = row[EXIT_TAG_IDX]
                 # Custom exit pricing only for exit-signals
                 if order_type == 'limit':
-                    closerate = strategy_safe_wrapper(self.strategy.custom_exit_price,
+                    close_rate = strategy_safe_wrapper(self.strategy.custom_exit_price,
-                                                      default_retval=closerate)(
+                                                       default_retval=close_rate)(
                         pair=trade.pair, trade=trade,
                         current_time=exit_candle_time,
-                        proposed_rate=closerate, current_profit=current_profit,
+                        proposed_rate=close_rate, current_profit=current_profit,
                         exit_tag=exit_reason)
                     # We can't place orders lower than current low.
                     # freqtrade does not support this in live, and the order would fill immediately
                     if trade.is_short:
-                        closerate = min(closerate, row[HIGH_IDX])
+                        close_rate = min(close_rate, row[HIGH_IDX])
                     else:
-                        closerate = max(closerate, row[LOW_IDX])
+                        close_rate = max(close_rate, row[LOW_IDX])
             # Confirm trade exit:
             time_in_force = self.strategy.order_time_in_force['exit']
 
             if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)(
                     pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount,
-                    rate=closerate,
+                    rate=close_rate,
                     time_in_force=time_in_force,
                     sell_reason=exit_reason,  # deprecated
                     exit_reason=exit_reason,
@@ -600,12 +600,12 @@ class Backtesting:
                 side=trade.exit_side,
                 order_type=order_type,
                 status="open",
-                price=closerate,
+                price=close_rate,
-                average=closerate,
+                average=close_rate,
                 amount=trade.amount,
                 filled=0,
                 remaining=trade.amount,
-                cost=trade.amount * closerate,
+                cost=trade.amount * close_rate,
             )
             trade.orders.append(order)
             return trade

freqtrade/rpc/telegram.py

@@ -1417,7 +1417,7 @@ class Telegram(RPCHandler):
             "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n"
             "*/forceexit <trade_id>|all:* `Instantly exits the given trade or all trades, "
             "regardless of profit`\n"
-            "*/fe <trade_id>|all:* `Alias to /forceexit`"
+            "*/fe <trade_id>|all:* `Alias to /forceexit`\n"
             f"{force_enter_text if self._config.get('force_entry_enable', False) else ''}"
             "*/delete <trade_id>:* `Instantly delete the given trade in the database`\n"
             "*/whitelist:* `Show current whitelist` \n"

requirements-plot.txt

@@ -1,4 +1,4 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==5.7.0
+plotly==5.8.0

setup.py

@@ -42,7 +42,7 @@ setup(
     ],
     install_requires=[
         # from requirements.txt
-        'ccxt>=1.79.69',
+        'ccxt>=1.80.67',
         'SQLAlchemy',
         'python-telegram-bot>=13.4',
         'arrow>=0.17.0',

tests/data/test_history.py

@@ -158,21 +158,22 @@ def test_testdata_path(testdatadir) -> None:
     assert str(Path('tests') / 'testdata') in str(testdatadir)
 
 
-@pytest.mark.parametrize("pair,expected_result,candle_type", [
+@pytest.mark.parametrize("pair,timeframe,expected_result,candle_type", [
-    ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json', ""),
+    ("ETH/BTC", "5m", "freqtrade/hello/world/ETH_BTC-5m.json", ""),
-    ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json', ""),
+    ("ETH/USDT", "1M", "freqtrade/hello/world/ETH_USDT-1Mo.json", ""),
-    ("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json', ""),
+    ("Fabric Token/ETH", "5m", "freqtrade/hello/world/Fabric_Token_ETH-5m.json", ""),
-    (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json', ""),
+    ("ETHH20", "5m", "freqtrade/hello/world/ETHH20-5m.json", ""),
-    ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json', ""),
+    (".XBTBON2H", "5m", "freqtrade/hello/world/_XBTBON2H-5m.json", ""),
-    ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json', ""),
+    ("ETHUSD.d", "5m", "freqtrade/hello/world/ETHUSD_d-5m.json", ""),
-    ("ETH/BTC", 'freqtrade/hello/world/futures/ETH_BTC-5m-mark.json', "mark"),
+    ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/ACC_OLD_BTC-5m.json", ""),
-    ("ACC_OLD/BTC", 'freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json', "index"),
+    ("ETH/BTC", "5m", "freqtrade/hello/world/futures/ETH_BTC-5m-mark.json", "mark"),
+    ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json", "index"),
 ])
-def test_json_pair_data_filename(pair, expected_result, candle_type):
+def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type):
     fn = JsonDataHandler._pair_data_filename(
         Path('freqtrade/hello/world'),
         pair,
-        '5m',
+        timeframe,
         CandleType.from_string(candle_type)
     )
     assert isinstance(fn, Path)
@@ -180,7 +181,7 @@ def test_json_pair_data_filename(pair, expected_result, candle_type):
     fn = JsonGzDataHandler._pair_data_filename(
         Path('freqtrade/hello/world'),
         pair,
-        '5m',
+        timeframe,
         candle_type=CandleType.from_string(candle_type)
     )
     assert isinstance(fn, Path)

tests/exchange/test_ccxt_precise.py

@@ -0,0 +1,75 @@
+from ccxt import Precise
+
+
+ws = Precise('-1.123e-6')
+ws = Precise('-1.123e-6')
+xs = Precise('0.00000002')
+ys = Precise('69696900000')
+zs = Precise('0')
+
+
+def test_precise():
+    assert ys * xs == '1393.938'
+    assert xs * ys == '1393.938'
+
+    assert ys + xs == '69696900000.00000002'
+    assert xs + ys == '69696900000.00000002'
+    assert xs - ys == '-69696899999.99999998'
+    assert ys - xs == '69696899999.99999998'
+    assert xs / ys == '0'
+    assert ys / xs == '3484845000000000000'
+
+    assert ws * xs == '-0.00000000000002246'
+    assert xs * ws == '-0.00000000000002246'
+
+    assert ws + xs == '-0.000001103'
+    assert xs + ws == '-0.000001103'
+
+    assert xs - ws == '0.000001143'
+    assert ws - xs == '-0.000001143'
+
+    assert xs / ws == '-0.017809439002671415'
+    assert ws / xs == '-56.15'
+
+    assert zs * ws == '0'
+    assert zs * xs == '0'
+    assert zs * ys == '0'
+    assert ws * zs == '0'
+    assert xs * zs == '0'
+    assert ys * zs == '0'
+
+    assert zs + ws == '-0.000001123'
+    assert zs + xs == '0.00000002'
+    assert zs + ys == '69696900000'
+    assert ws + zs == '-0.000001123'
+    assert xs + zs == '0.00000002'
+    assert ys + zs == '69696900000'
+
+    assert abs(Precise('-500.1')) == '500.1'
+    assert abs(Precise('213')) == '213'
+
+    assert abs(Precise('-500.1')) == '500.1'
+    assert -Precise('213') == '-213'
+
+    assert Precise('10.1') % Precise('0.5') == '0.1'
+    assert Precise('5550') % Precise('120') == '30'
+
+    assert Precise('-0.0') == Precise('0')
+    assert Precise('5.534000') == Precise('5.5340')
+
+    assert min(Precise('-3.1415'), Precise('-2')) == '-3.1415'
+
+    assert max(Precise('3.1415'), Precise('-2')) == '3.1415'
+
+    assert Precise('2') > Precise('1.2345')
+    assert not Precise('-3.1415') > Precise('-2')
+    assert not Precise('3.1415') > Precise('3.1415')
+    assert Precise.string_gt('3.14150000000000000000001', '3.1415')
+
+    assert Precise('3.1415') >= Precise('3.1415')
+    assert Precise('3.14150000000000000000001') >= Precise('3.1415')
+
+    assert not Precise('3.1415') < Precise('3.1415')
+
+    assert Precise('3.1415') <= Precise('3.1415')
+    assert Precise('3.1415') <= Precise('3.14150000000000000000001')

tests/exchange/test_exchange.py

@@ -305,6 +305,7 @@ def test_amount_to_precision(
     (234.53, 4, 0.5, 235.0),
     (0.891534, 4, 0.0001, 0.8916),
     (64968.89, 4, 0.01, 64968.89),
+    (0.000000003483, 4, 1e-12, 0.000000003483),
 
 ])
 def test_price_to_precision(default_conf, mocker, price, precision_mode, precision, expected):