Merge branch 'mark-price-candles' of https://github.com/samgermain/freqtrade into mark-price-candles

docs/configuration.md

@@ -204,9 +204,8 @@ There are several methods to configure how much of the stake currency the bot wi
 #### Minimum trade stake
 
 The minimum stake amount will depend on exchange and pair and is usually listed in the exchange support pages.
-Assuming the minimum tradable amount for XRP/USD is 20 XRP (given by the exchange), and the price is 0.6$.
 
-The minimum stake amount to buy this pair is, therefore, `20 * 0.6 ~= 12`.
+Assuming the minimum tradable amount for XRP/USD is 20 XRP (given by the exchange), and the price is 0.6$, the minimum stake amount to buy this pair is `20 * 0.6 ~= 12`.
 This exchange has also a limit on USD - where all orders must be > 10$ - which however does not apply in this case.
 
 To guarantee safe execution, freqtrade will not allow buying with a stake-amount of 10.1$, instead, it'll make sure that there's enough space to place a stoploss below the pair (+ an offset, defined by `amount_reserve_percent`, which defaults to 5%).

docs/includes/pairlists.md

@@ -292,7 +292,7 @@ If the trading range over the last 10 days is <1% or >99%, remove the pair from
 
 #### VolatilityFilter
 
-Volatility is the degree of historical variation of a pairs over time, is is measured by the standard deviation of logarithmic daily returns. Returns are assumed to be normally distributed, although actual distribution might be different. In a normal distribution, 68% of observations fall within one standard deviation and 95% of observations fall within two standard deviations. Assuming a volatility of 0.05 means that the expected returns for 20 out of 30 days is expected to be less than 5% (one standard deviation). Volatility is a positive ratio of the expected deviation of return and can be greater than 1.00. Please refer to the wikipedia definition of [`volatility`](https://en.wikipedia.org/wiki/Volatility_(finance)).
+Volatility is the degree of historical variation of a pairs over time, it is measured by the standard deviation of logarithmic daily returns. Returns are assumed to be normally distributed, although actual distribution might be different. In a normal distribution, 68% of observations fall within one standard deviation and 95% of observations fall within two standard deviations. Assuming a volatility of 0.05 means that the expected returns for 20 out of 30 days is expected to be less than 5% (one standard deviation). Volatility is a positive ratio of the expected deviation of return and can be greater than 1.00. Please refer to the wikipedia definition of [`volatility`](https://en.wikipedia.org/wiki/Volatility_(finance)).
 
 This filter removes pairs if the average volatility over a `lookback_days` days is below `min_volatility` or above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
 

freqtrade/exchange/binance.py

@@ -3,7 +3,7 @@ import json
 import logging
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
 
 import arrow
 import ccxt
@@ -119,10 +119,6 @@ class Binance(Exchange):
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
 
-    def market_is_future(self, market: Dict[str, Any]) -> bool:
-        # TODO-lev: This should be unified in ccxt to "swap"...
-        return market.get('future', False) is True
-
     @retrier
     def fill_leverage_brackets(self):
         """
@@ -212,9 +208,9 @@ class Binance(Exchange):
         """
         if is_new_pair:
             x = await self._async_get_candle_history(pair, timeframe, 0, candle_type)
-            if x and x[2] and x[2][0] and x[2][0][0] > since_ms:
+            if x and x[3] and x[3][0] and x[3][0][0] > since_ms:
                 # Set starting date to first available candle.
-                since_ms = x[2][0][0]
+                since_ms = x[3][0][0]
                 logger.info(f"Candle-data for {pair} available starting with "
                             f"{arrow.get(since_ms // 1000).isoformat()}.")
 

freqtrade/exchange/exchange.py

@@ -338,7 +338,7 @@ class Exchange:
         return self.markets.get(pair, {}).get('base', '')
 
     def market_is_future(self, market: Dict[str, Any]) -> bool:
-        return market.get('swap', False) is True
+        return market.get(self._ft_has["ccxt_futures_name"], False) is True
 
     def market_is_spot(self, market: Dict[str, Any]) -> bool:
         return market.get('spot', False) is True
@@ -1419,7 +1419,7 @@ class Exchange:
                         pair, timeframe, since_ms=since_ms, candle_type=candle_type))
             else:
                 logger.debug(
-                    "Using cached candle (OHLCV) data for pair %s, timeframe %s ...",
+                    "Using cached candle (OHLCV) data for pair %s, timeframe %s, candleType %s ...",
                     pair, timeframe, candle_type
                 )
                 cached_pairs.append((pair, timeframe, candle_type))

freqtrade/exchange/ftx.py

@@ -20,7 +20,8 @@ class Ftx(Exchange):
     _ft_has: Dict = {
         "stoploss_on_exchange": True,
         "ohlcv_candle_limit": 1500,
-        "mark_ohlcv_price": "index"
+        "mark_ohlcv_price": "index",
+        "ccxt_futures_name": "future"
     }
 
     _supported_trading_mode_collateral_pairs: List[Tuple[TradingMode, Collateral]] = [
@@ -159,7 +160,3 @@ class Ftx(Exchange):
         if order['type'] == 'stop':
             return safe_value_fallback2(order, order, 'id_stop', 'id')
         return order['id']
-
-    def market_is_future(self, market: Dict[str, Any]) -> bool:
-        # TODO-lev: This should be unified in ccxt to "swap"...
-        return market.get('future', False) is True

freqtrade/optimize/backtesting.py

@@ -70,7 +70,7 @@ class Backtesting:
         self.all_results: Dict[str, Dict] = {}
         self._exchange_name = self.config['exchange']['name']
         self.exchange = ExchangeResolver.load_exchange(self._exchange_name, self.config)
-        self.dataprovider = DataProvider(self.config, None)
+        self.dataprovider = DataProvider(self.config, self.exchange)
 
         if self.config.get('strategy_list', None):
             for strat in list(self.config['strategy_list']):

freqtrade/rpc/api_server/api_schemas.py

@@ -144,6 +144,7 @@ class OrderTypes(BaseModel):
 
 class ShowConfig(BaseModel):
     version: str
+    api_version: float
     dry_run: bool
     trading_mode: str
     short_allowed: bool

freqtrade/rpc/api_server/api_v1.py

@@ -26,6 +26,11 @@ from freqtrade.rpc.rpc import RPCException
 
 logger = logging.getLogger(__name__)
 
+# API version
+# Pre-1.1, no version was provided
+# Version increments should happen in "small" steps (1.1, 1.12, ...) unless big changes happen.
+API_VERSION = 1.1
+
 # Public API, requires no auth.
 router_public = APIRouter()
 # Private API, protected by authentication
@@ -117,7 +122,9 @@ def show_config(rpc: Optional[RPC] = Depends(get_rpc_optional), config=Depends(g
     state = ''
     if rpc:
         state = rpc._freqtrade.state
-    return RPC._rpc_show_config(config, state)
+    resp = RPC._rpc_show_config(config, state)
+    resp['api_version'] = API_VERSION
+    return resp
 
 
 @router.post('/forcebuy', response_model=ForceBuyResponse, tags=['trading'])

freqtrade/rpc/telegram.py

@@ -274,11 +274,11 @@ class Telegram(RPCHandler):
             f"*Buy Tag:* `{msg['buy_tag']}`\n"
             f"*Sell Reason:* `{msg['sell_reason']}`\n"
             f"*Duration:* `{msg['duration']} ({msg['duration_min']:.1f} min)`\n"
-            f"*Amount:* `{msg['amount']:.8f}`\n")
+            f"*Amount:* `{msg['amount']:.8f}`\n"
+            f"*Open Rate:* `{msg['open_rate']:.8f}`\n")
 
         if msg['type'] == RPCMessageType.SELL:
-            message += (f"*Open Rate:* `{msg['open_rate']:.8f}`\n"
+            message += (f"*Current Rate:* `{msg['current_rate']:.8f}`\n"
-                        f"*Current Rate:* `{msg['current_rate']:.8f}`\n"
                         f"*Close Rate:* `{msg['limit']:.8f}`")
 
         elif msg['type'] == RPCMessageType.SELL_FILL:

freqtrade/strategy/informative_decorator.py

@@ -81,12 +81,11 @@ def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, metadata:
         # Not specifying an asset will define informative dataframe for current pair.
         asset = metadata['pair']
 
-    if '/' in asset:
+    market = strategy.dp.market(asset)
-        base, quote = asset.split('/')
+    if market is None:
-    else:
+        raise OperationalException(f'Market {asset} is not available.')
-        # When futures are supported this may need reevaluation.
+    base = market['base']
-        # base, quote = asset, ''
+    quote = market['quote']
-        raise OperationalException('Not implemented.')
 
     # Default format. This optimizes for the common case: informative pairs using same stake
     # currency. When quote currency matches stake currency, column name will omit base currency.

requirements-dev.txt

@@ -20,7 +20,7 @@ time-machine==2.4.0
 nbconvert==6.3.0
 
 # mypy types
-types-cachetools==4.2.4
+types-cachetools==4.2.5
 types-filelock==3.2.1
 types-requests==2.26.0
 types-tabulate==0.8.3

requirements-hyperopt.txt

@@ -5,7 +5,7 @@
 scipy==1.7.2
 scikit-learn==1.0.1
 scikit-optimize==0.9.0
-filelock==3.3.2
+filelock==3.4.0
 joblib==1.1.0
 psutil==5.8.0
 progressbar2==3.55.0

requirements-plot.txt

@@ -1,5 +1,5 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==5.3.1
+plotly==5.4.0
 

requirements.txt

@@ -2,10 +2,10 @@ numpy==1.21.4
 pandas==1.3.4
 pandas-ta==0.3.14b
 
-ccxt==1.61.24
+ccxt==1.61.92
 # Pin cryptography for now due to rust build errors with piwheels
-cryptography==35.0.0
+cryptography==36.0.0
-aiohttp==3.7.4.post0
+aiohttp==3.8.1
 SQLAlchemy==1.4.27
 python-telegram-bot==13.8.1
 arrow==1.2.1

tests/exchange/test_exchange.py

@@ -1748,13 +1748,13 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None:
 
     assert exchange._api_async.fetch_ohlcv.call_count == 0
     assert log_has(f"Using cached candle (OHLCV) data for pair {pairs[0][0]}, "
-                   f"timeframe {pairs[0][1]} ...",
+                   f"timeframe {pairs[0][1]}, candleType  ...",
                    caplog)
     res = exchange.refresh_latest_ohlcv(
         [('IOTA/ETH', '5m', ''), ('XRP/ETH', '5m', ''), ('XRP/ETH', '1d', '')],
         cache=False
     )
-    assert len(res) == 4
+    assert len(res) == 3
 
 
 @pytest.mark.asyncio
@@ -3329,7 +3329,7 @@ def test_validate_trading_mode_and_collateral(
     ("bibox", "margin", {"has": {"fetchCurrencies": False}, "options": {"defaultType": "margin"}}),
     ("bibox", "futures", {"has": {"fetchCurrencies": False}, "options": {"defaultType": "swap"}}),
     ("bybit", "futures", {"options": {"defaultType": "linear"}}),
-    ("ftx", "futures", {"options": {"defaultType": "swap"}}),
+    ("ftx", "futures", {"options": {"defaultType": "future"}}),
     ("gateio", "futures", {"options": {"defaultType": "swap"}}),
     ("hitbtc", "futures", {"options": {"defaultType": "swap"}}),
     ("kraken", "futures", {"options": {"defaultType": "swap"}}),

tests/rpc/test_rpc_apiserver.py

@@ -541,6 +541,8 @@ def test_api_show_config(botclient):
     assert 'ask_strategy' in response
     assert 'unfilledtimeout' in response
     assert 'version' in response
+    assert 'api_version' in response
+    assert 1.1 <= response['api_version'] <= 1.2
 
 
 def test_api_daily(botclient, mocker, ticker, fee, markets):

tests/rpc/test_rpc_telegram.py

@@ -1847,6 +1847,7 @@ def test_send_msg_sell_fill_notification(default_conf, mocker) -> None:
             '*Sell Reason:* `stop_loss`\n'
             '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n'
             '*Amount:* `1333.33333333`\n'
+            '*Open Rate:* `0.00007500`\n'
             '*Close Rate:* `0.00003201`'
             )
 

tests/strategy/strats/informative_decorator_strategy.py

@@ -19,7 +19,7 @@ class InformativeDecoratorTest(IStrategy):
     startup_candle_count: int = 20
 
     def informative_pairs(self):
-        return [('BTC/USDT', '5m', '')]
+        return [('NEO/USDT', '5m', '')]
 
     def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         dataframe['buy'] = 0
@@ -37,8 +37,8 @@ class InformativeDecoratorTest(IStrategy):
         return dataframe
 
     # Simple informative test.
-    @informative('1h', 'BTC/{stake}')
+    @informative('1h', 'NEO/{stake}')
-    def populate_indicators_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+    def populate_indicators_neo_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         dataframe['rsi'] = 14
         return dataframe
 
@@ -49,7 +49,7 @@ class InformativeDecoratorTest(IStrategy):
         return dataframe
 
     # Formatting test.
-    @informative('30m', 'BTC/{stake}', '{column}_{BASE}_{QUOTE}_{base}_{quote}_{asset}_{timeframe}')
+    @informative('30m', 'NEO/{stake}', '{column}_{BASE}_{QUOTE}_{base}_{quote}_{asset}_{timeframe}')
     def populate_indicators_btc_1h_2(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         dataframe['rsi'] = 14
         return dataframe
@@ -67,7 +67,7 @@ class InformativeDecoratorTest(IStrategy):
         dataframe['rsi_less'] = dataframe['rsi'] < dataframe['rsi_1h']
 
         # Mixing manual informative pairs with decorators.
-        informative = self.dp.get_pair_dataframe('BTC/USDT', '5m', '')
+        informative = self.dp.get_pair_dataframe('NEO/USDT', '5m', '')
         informative['rsi'] = 14
         dataframe = merge_informative_pair(dataframe, informative, self.timeframe, '5m', ffill=True)
 

tests/strategy/test_strategy_helpers.py

@@ -7,6 +7,7 @@ import pytest
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.strategy import (merge_informative_pair, stoploss_from_absolute, stoploss_from_open,
                                 timeframe_to_minutes)
+from tests.conftest import get_patched_exchange
 
 
 def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'):
@@ -156,9 +157,9 @@ def test_informative_decorator(mocker, default_conf):
         ('LTC/USDT', '5m', ''): test_data_5m,
         ('LTC/USDT', '30m', ''): test_data_30m,
         ('LTC/USDT', '1h', ''): test_data_1h,
-        ('BTC/USDT', '30m', ''): test_data_30m,
+        ('NEO/USDT', '30m', ''): test_data_30m,
-        ('BTC/USDT', '5m', ''): test_data_5m,
+        ('NEO/USDT', '5m', ''): test_data_5m,
-        ('BTC/USDT', '1h', ''): test_data_1h,
+        ('NEO/USDT', '1h', ''): test_data_1h,
         ('ETH/USDT', '1h', ''): test_data_1h,
         ('ETH/USDT', '30m', ''): test_data_30m,
         ('ETH/BTC', '1h', ''): test_data_1h,
@@ -166,15 +167,16 @@ def test_informative_decorator(mocker, default_conf):
     from .strats.informative_decorator_strategy import InformativeDecoratorTest
     default_conf['stake_currency'] = 'USDT'
     strategy = InformativeDecoratorTest(config=default_conf)
-    strategy.dp = DataProvider({}, None, None)
+    exchange = get_patched_exchange(mocker, default_conf)
+    strategy.dp = DataProvider({}, exchange, None)
     mocker.patch.object(strategy.dp, 'current_whitelist', return_value=[
-        'XRP/USDT', 'LTC/USDT', 'BTC/USDT'
+        'XRP/USDT', 'LTC/USDT', 'NEO/USDT'
     ])
 
     assert len(strategy._ft_informative) == 6   # Equal to number of decorators used
     informative_pairs = [('XRP/USDT', '1h', ''), ('LTC/USDT', '1h', ''), ('XRP/USDT', '30m', ''),
-                         ('LTC/USDT', '30m', ''), ('BTC/USDT', '1h', ''), ('BTC/USDT', '30m', ''),
+                         ('LTC/USDT', '30m', ''), ('NEO/USDT', '1h', ''), ('NEO/USDT', '30m', ''),
-                         ('BTC/USDT', '5m', ''), ('ETH/BTC', '1h', ''), ('ETH/USDT', '30m', '')]
+                         ('NEO/USDT', '5m', ''), ('ETH/BTC', '1h', ''), ('ETH/USDT', '30m', '')]
     for inf_pair in informative_pairs:
         assert inf_pair in strategy.gather_informative_pairs()
 
@@ -187,8 +189,8 @@ def test_informative_decorator(mocker, default_conf):
         {p: data[(p, strategy.timeframe, '')] for p in ('XRP/USDT', 'LTC/USDT')})
     expected_columns = [
         'rsi_1h', 'rsi_30m',                    # Stacked informative decorators
-        'btc_usdt_rsi_1h',                      # BTC 1h informative
+        'neo_usdt_rsi_1h',                      # NEO 1h informative
-        'rsi_BTC_USDT_btc_usdt_BTC/USDT_30m',   # Column formatting
+        'rsi_NEO_USDT_neo_usdt_NEO/USDT_30m',   # Column formatting
         'rsi_from_callable',                    # Custom column formatter
         'eth_btc_rsi_1h',                       # Quote currency not matching stake currency
         'rsi', 'rsi_less',                      # Non-informative columns