Merge branch 'feat/short' into pr/samgermain/5780

docs/configuration.md

@@ -128,9 +128,10 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
 | `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
 | `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
+| `exchange.uid` | API uid to use for the exchange. Only required when you are in production mode and for exchanges that use uid for API requests.<br>**Keep it in secret, do not disclose publicly.** <br> **Datatype:** String
 | `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Supports regex pairs as `.*/BTC`. Not used by VolumePairList. [More information](plugins.md#pairlists-and-pairlist-handlers). <br> **Datatype:** List
 | `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting. [More information](plugins.md#pairlists-and-pairlist-handlers). <br> **Datatype:** List
-| `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) <br> **Datatype:** Dict
+| `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for additional ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation). Please avoid adding exchange secrets here (use the dedicated fields instead), as they may be contained in logs. <br> **Datatype:** Dict
 | `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) <br> **Datatype:** Dict
 | `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange  and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) <br> **Datatype:** Dict
 | `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded. <br>*Defaults to `60` minutes.* <br> **Datatype:** Positive Integer

freqtrade/data/history/hdf5datahandler.py

@@ -6,7 +6,6 @@ from typing import List, Optional
 import numpy as np
 import pandas as pd
 
-from freqtrade import misc
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS,
                                  ListPairsWithTimeframes, TradeList)
@@ -121,20 +120,6 @@ class HDF5DataHandler(IDataHandler):
                                           'low': 'float', 'close': 'float', 'volume': 'float'})
         return pairdata
 
-    def ohlcv_purge(self, pair: str, timeframe: str, candle_type: str = '') -> bool:
-        """
-        Remove data for this pair
-        :param pair: Delete data for this pair.
-        :param timeframe: Timeframe (e.g. "5m")
-        :param candle_type: '', mark, index, premiumIndex, or funding_rate
-        :return: True when deleted, false if file did not exist.
-        """
-        filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)
-        if filename.exists():
-            filename.unlink()
-            return True
-        return False
-
     def ohlcv_append(
         self,
         pair: str,
@@ -210,17 +195,9 @@ class HDF5DataHandler(IDataHandler):
         trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None})
         return trades.values.tolist()
 
-    def trades_purge(self, pair: str) -> bool:
+    @classmethod
-        """
+    def _get_file_extension(cls):
-        Remove data for this pair
+        return "h5"
-        :param pair: Delete data for this pair.
-        :return: True when deleted, false if file did not exist.
-        """
-        filename = self._pair_trades_filename(self._datadir, pair)
-        if filename.exists():
-            filename.unlink()
-            return True
-        return False
 
     @classmethod
     def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str:
@@ -229,23 +206,3 @@ class HDF5DataHandler(IDataHandler):
     @classmethod
     def _pair_trades_key(cls, pair: str) -> str:
         return f"{pair}/trades"
-
-    @classmethod
-    def _pair_data_filename(
-        cls,
-        datadir: Path,
-        pair: str,
-        timeframe: str,
-        candle_type: str = ''
-    ) -> Path:
-        pair_s = misc.pair_to_filename(pair)
-        if candle_type:
-            candle_type = f"-{candle_type}"
-        filename = datadir.joinpath(f'{pair_s}-{timeframe}{candle_type}.h5')
-        return filename
-
-    @classmethod
-    def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
-        pair_s = misc.pair_to_filename(pair)
-        filename = datadir.joinpath(f'{pair_s}-trades.h5')
-        return filename

freqtrade/data/history/idatahandler.py

@@ -13,6 +13,7 @@ from typing import List, Optional, Type
 
 from pandas import DataFrame
 
+from freqtrade import misc
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import ListPairsWithTimeframes, TradeList
 from freqtrade.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe
@@ -29,6 +30,13 @@ class IDataHandler(ABC):
     def __init__(self, datadir: Path) -> None:
         self._datadir = datadir
 
+    @classmethod
+    def _get_file_extension(cls) -> str:
+        """
+        Get file extension for this particular datahandler
+        """
+        raise NotImplementedError()
+
     @abstractclassmethod
     def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
         """
@@ -83,7 +91,6 @@ class IDataHandler(ABC):
         :return: DataFrame with ohlcv data, or empty DataFrame
         """
 
-    @abstractmethod
     def ohlcv_purge(self, pair: str, timeframe: str, candle_type: str = '') -> bool:
         """
         Remove data for this pair
@@ -92,6 +99,11 @@ class IDataHandler(ABC):
         :param candle_type: '', mark, index, premiumIndex, or funding_rate
         :return: True when deleted, false if file did not exist.
         """
+        filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)
+        if filename.exists():
+            filename.unlink()
+            return True
+        return False
 
     @abstractmethod
     def ohlcv_append(
@@ -144,13 +156,17 @@ class IDataHandler(ABC):
         :return: List of trades
         """
 
-    @abstractmethod
     def trades_purge(self, pair: str) -> bool:
         """
         Remove data for this pair
         :param pair: Delete data for this pair.
         :return: True when deleted, false if file did not exist.
         """
+        filename = self._pair_trades_filename(self._datadir, pair)
+        if filename.exists():
+            filename.unlink()
+            return True
+        return False
 
     def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
         """
@@ -162,6 +178,26 @@ class IDataHandler(ABC):
         """
         return trades_remove_duplicates(self._trades_load(pair, timerange=timerange))
 
+    @classmethod
+    def _pair_data_filename(
+        cls,
+        datadir: Path,
+        pair: str,
+        timeframe: str,
+        candle_type: str = ''
+    ) -> Path:
+        pair_s = misc.pair_to_filename(pair)
+        if candle_type:
+            candle_type = f"-{candle_type}"
+        filename = datadir.joinpath(f'{pair_s}-{timeframe}{candle_type}.{cls._get_file_extension()}')
+        return filename
+
+    @classmethod
+    def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
+        pair_s = misc.pair_to_filename(pair)
+        filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
+        return filename
+
     @staticmethod
     def rebuild_pair_from_filename(pair: str) -> str:
         """

freqtrade/data/history/jsondatahandler.py

@@ -198,43 +198,10 @@ class JsonDataHandler(IDataHandler):
             pass
         return tradesdata
 
-    def trades_purge(self, pair: str) -> bool:
-        """
-        Remove data for this pair
-        :param pair: Delete data for this pair.
-        :return: True when deleted, false if file did not exist.
-        """
-        filename = self._pair_trades_filename(self._datadir, pair)
-        if filename.exists():
-            filename.unlink()
-            return True
-        return False
-
-    @classmethod
-    def _pair_data_filename(
-        cls,
-        datadir: Path,
-        pair: str,
-        timeframe: str,
-        candle_type: str = ''
-    ) -> Path:
-        pair_s = misc.pair_to_filename(pair)
-        if candle_type:
-            candle_type = f"-{candle_type}"
-        filename = datadir.joinpath(
-            f'{pair_s}-{timeframe}{candle_type}.{cls._get_file_extension()}')
-        return filename
-
     @classmethod
     def _get_file_extension(cls):
         return "json.gz" if cls._use_zip else "json"
 
-    @classmethod
-    def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
-        pair_s = misc.pair_to_filename(pair)
-        filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
-        return filename
-
 
 class JsonGzDataHandler(JsonDataHandler):
 

freqtrade/exchange/exchange.py

@@ -770,16 +770,20 @@ class Exchange:
         if not self.exchange_has('fetchL2OrderBook'):
             return True
         ob = self.fetch_l2_order_book(pair, 1)
-        if side == 'buy':
+        try:
-            price = ob['asks'][0][0]
+            if side == 'buy':
-            logger.debug(f"{pair} checking dry buy-order: price={price}, limit={limit}")
+                price = ob['asks'][0][0]
-            if limit >= price:
+                logger.debug(f"{pair} checking dry buy-order: price={price}, limit={limit}")
-                return True
+                if limit >= price:
-        else:
+                    return True
-            price = ob['bids'][0][0]
+            else:
-            logger.debug(f"{pair} checking dry sell-order: price={price}, limit={limit}")
+                price = ob['bids'][0][0]
-            if limit <= price:
+                logger.debug(f"{pair} checking dry sell-order: price={price}, limit={limit}")
-                return True
+                if limit <= price:
+                    return True
+        except IndexError:
+            # Ignore empty orderbooks when filling - can be filled with the next iteration.
+            pass
         return False
 
     def check_dry_limit_order_filled(self, order: Dict[str, Any]) -> Dict[str, Any]:

tests/exchange/test_exchange.py

@@ -1064,6 +1064,12 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
     assert order_closed['status'] == 'closed'
     assert order['fee']
 
+    # Empty orderbook test
+    mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book',
+                 return_value={'asks': [], 'bids': []})
+    exchange._dry_run_open_orders[order['id']]['status'] = 'open'
+    order_closed = exchange.fetch_dry_run_order(order['id'])
+
 
 @pytest.mark.parametrize("side,rate,amount,endprice", [
     # spread is 25.263-25.266