Merge branch 'develop' into feat/short

README.md

@@ -30,6 +30,7 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even
 - [X] [Bittrex](https://bittrex.com/)
 - [X] [Kraken](https://kraken.com/)
 - [X] [FTX](https://ftx.com)
+- [X] [Gate.io](https://www.gate.io/ref/6266643)
 - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_
 
 ### Community tested

docs/exchanges.md

@@ -58,6 +58,12 @@ Bittrex does not support market orders. If you have a message at the bot startup
 Bittrex also does not support `VolumePairlist` due to limited / split API constellation at the moment.
 Please use `StaticPairlist`. Other pairlists (other than `VolumePairlist`) should not be affected.
 
+### Volume pairlist
+
+Bittrex does not support the direct usage of VolumePairList. This can however be worked around by using the advanced mode with `lookback_days: 1` (or more), which will emulate 24h volume.
+
+Read more in the [pairlist documentation](plugins.md#volumepairlist-advanced-mode).
+
 ### Restricted markets
 
 Bittrex split its exchange into US and International versions.

docs/includes/pairlists.md

@@ -82,6 +82,8 @@ Filtering instances (not the first position in the list) will not apply any cach
 
 You can define a minimum volume with `min_value` - which will filter out pairs with a volume lower than the specified value in the specified timerange.
 
+### VolumePairList Advanced mode
+
 `VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a  more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles.
 
 For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days:
@@ -105,6 +107,24 @@ For convenience `lookback_days` can be specified, which will imply that 1d candl
 !!! Warning "Performance implications when using lookback range"
     If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation.
 
+??? Tip "Unsupported exchanges (Bittrex, Gemini)"
+    On some exchanges (like Bittrex and Gemini), regular VolumePairList does not work as the api does not natively provide 24h volume. This can be worked around by using candle data to build the volume.
+    To roughly simulate 24h volume, you can use the following configuration.
+    Please note that These pairlists will only refresh once per day.
+
+    ```json
+    "pairlists": [
+        {
+            "method": "VolumePairList",
+            "number_assets": 20,
+            "sort_key": "quoteVolume",
+            "min_value": 0,
+            "refresh_period": 86400,
+            "lookback_days": 1
+        }
+    ],
+    ```
+
 More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles:
 
 ```json

docs/index.md

@@ -40,6 +40,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual,
 - [X] [Bittrex](https://bittrex.com/)
 - [X] [FTX](https://ftx.com)
 - [X] [Kraken](https://kraken.com/)
+- [X] [Gate.io](https://www.gate.io/ref/6266643)
 - [ ] [potentially many others through <img alt="ccxt" width="30px" src="assets/ccxt-logo.svg" />](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_
 
 ### Community tested

freqtrade/configuration/__init__.py

@@ -1,6 +1,6 @@
 # flake8: noqa: F401
 
-from freqtrade.configuration.check_exchange import check_exchange, remove_credentials
+from freqtrade.configuration.check_exchange import check_exchange
 from freqtrade.configuration.config_setup import setup_utils_configuration
 from freqtrade.configuration.config_validation import validate_config_consistency
 from freqtrade.configuration.configuration import Configuration

freqtrade/configuration/check_exchange.py

@@ -10,19 +10,6 @@ from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt,
 logger = logging.getLogger(__name__)
 
 
-def remove_credentials(config: Dict[str, Any]) -> None:
-    """
-    Removes exchange keys from the configuration and specifies dry-run
-    Used for backtesting / hyperopt / edge and utils.
-    Modifies the input dict!
-    """
-    config['exchange']['key'] = ''
-    config['exchange']['secret'] = ''
-    config['exchange']['password'] = ''
-    config['exchange']['uid'] = ''
-    config['dry_run'] = True
-
-
 def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
     """
     Check if the exchange name in the config file is supported by Freqtrade

freqtrade/configuration/config_setup.py

@@ -3,7 +3,6 @@ from typing import Any, Dict
 
 from freqtrade.enums import RunMode
 
-from .check_exchange import remove_credentials
 from .config_validation import validate_config_consistency
 from .configuration import Configuration
 
@@ -21,8 +20,8 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str
     configuration = Configuration(args, method)
     config = configuration.get_config()
 
-    # Ensure we do not use Exchange credentials
+    # Ensure these modes are using Dry-run
-    remove_credentials(config)
+    config['dry_run'] = True
     validate_config_consistency(config)
 
     return config

freqtrade/data/history/history_utils.py

@@ -197,7 +197,8 @@ def _download_pair_history(pair: str, *,
                                                timeframe=timeframe,
                                                since_ms=since_ms if since_ms else
                                                arrow.utcnow().shift(
-                                                   days=-new_pairs_days).int_timestamp * 1000
+                                                   days=-new_pairs_days).int_timestamp * 1000,
+                                               is_new_pair=data.empty
                                                )
         # TODO: Maybe move parsing to exchange class (?)
         new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,

freqtrade/exchange/__init__.py

@@ -1,6 +1,6 @@
 # flake8: noqa: F401
 # isort: off
-from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS
+from freqtrade.exchange.common import remove_credentials, MAP_EXCHANGE_CHILDCLASS
 from freqtrade.exchange.exchange import Exchange
 # isort: on
 from freqtrade.exchange.bibox import Bibox

freqtrade/exchange/binance.py

@@ -1,7 +1,8 @@
 """ Binance exchange subclass """
 import logging
-from typing import Dict
+from typing import Dict, List
 
+import arrow
 import ccxt
 
 from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException,
@@ -90,3 +91,20 @@ class Binance(Exchange):
                 f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e
         except ccxt.BaseError as e:
             raise OperationalException(e) from e
+
+    async def _async_get_historic_ohlcv(self, pair: str, timeframe: str,
+                                        since_ms: int, is_new_pair: bool
+                                        ) -> List:
+        """
+        Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date
+        Does not work for other exchanges, which don't return the earliest data when called with "0"
+        """
+        if is_new_pair:
+            x = await self._async_get_candle_history(pair, timeframe, 0)
+            if x and x[2] and x[2][0] and x[2][0][0] > since_ms:
+                # Set starting date to first available candle.
+                since_ms = x[2][0][0]
+                logger.info(f"Candle-data for {pair} available starting with "
+                            f"{arrow.get(since_ms // 1000).isoformat()}.")
+        return await super()._async_get_historic_ohlcv(
+            pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair)

freqtrade/exchange/common.py

@@ -51,6 +51,19 @@ EXCHANGE_HAS_OPTIONAL = [
 ]
 
 
+def remove_credentials(config) -> None:
+    """
+    Removes exchange keys from the configuration and specifies dry-run
+    Used for backtesting / hyperopt / edge and utils.
+    Modifies the input dict!
+    """
+    if config.get('dry_run', False):
+        config['exchange']['key'] = ''
+        config['exchange']['secret'] = ''
+        config['exchange']['password'] = ''
+        config['exchange']['uid'] = ''
+
+
 def calculate_backoff(retrycount, max_retries):
     """
     Calculate backoff

freqtrade/exchange/exchange.py

@@ -26,9 +26,9 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun
                                   InvalidOrderException, OperationalException, PricingError,
                                   RetryableOrderError, TemporaryError)
 from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES,
-                                       EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier,
+                                       EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED,
-                                       retrier_async)
+                                       remove_credentials, retrier, retrier_async)
-from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
+from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 
 
@@ -104,6 +104,7 @@ class Exchange:
 
         # Holds all open sell orders for dry_run
         self._dry_run_open_orders: Dict[str, Any] = {}
+        remove_credentials(config)
 
         if config['dry_run']:
             logger.info('Instance is running with dry_run enabled')
@@ -1194,7 +1195,7 @@ class Exchange:
     # Historic data
 
     def get_historic_ohlcv(self, pair: str, timeframe: str,
-                           since_ms: int) -> List:
+                           since_ms: int, is_new_pair: bool = False) -> List:
         """
         Get candle history using asyncio and returns the list of candles.
         Handles all async work for this.
@@ -1206,7 +1207,7 @@ class Exchange:
         """
         return asyncio.get_event_loop().run_until_complete(
             self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe,
-                                           since_ms=since_ms))
+                                           since_ms=since_ms, is_new_pair=is_new_pair))
 
     def get_historic_ohlcv_as_df(self, pair: str, timeframe: str,
                                  since_ms: int) -> DataFrame:
@@ -1221,11 +1222,12 @@ class Exchange:
         return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True,
                                   drop_incomplete=self._ohlcv_partial_candle)
 
-    async def _async_get_historic_ohlcv(self, pair: str,
+    async def _async_get_historic_ohlcv(self, pair: str, timeframe: str,
-                                        timeframe: str,
+                                        since_ms: int, is_new_pair: bool
-                                        since_ms: int) -> List:
+                                        ) -> List:
         """
         Download historic ohlcv
+        :param is_new_pair: used by binance subclass to allow "fast" new pair downloading
         """
 
         one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe)
@@ -1238,10 +1240,11 @@ class Exchange:
             pair, timeframe, since) for since in
             range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)]
 
-        results = await asyncio.gather(*input_coroutines, return_exceptions=True)
-
-        # Combine gathered results
         data: List = []
+        # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling
+        for input_coro in chunks(input_coroutines, 100):
+
+            results = await asyncio.gather(*input_coro, return_exceptions=True)
             for res in results:
                 if isinstance(res, Exception):
                     logger.warning("Async code raised an exception: %s", res.__class__.__name__)
@@ -1252,7 +1255,7 @@ class Exchange:
                     data.extend(new_data)
         # Sort data again after extending the result - above calls return in "async order"
         data = sorted(data, key=lambda x: x[0])
-        logger.info("Downloaded data for %s with length %s.", pair, len(data))
+        logger.info(f"Downloaded data for {pair} with length {len(data)}.")
         return data
 
     def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *,

freqtrade/exchange/gateio.py

@@ -21,3 +21,5 @@ class Gateio(Exchange):
     _ft_has: Dict = {
         "ohlcv_candle_limit": 1000,
     }
+
+    _headers = {'X-Gate-Channel-Id': 'freqtrade'}

freqtrade/optimize/backtesting.py

@@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple
 
 from pandas import DataFrame
 
-from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
+from freqtrade.configuration import TimeRange, validate_config_consistency
 from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.data import history
 from freqtrade.data.btanalysis import trade_list_to_dataframe
@@ -61,8 +61,7 @@ class Backtesting:
         self.config = config
         self.results: Optional[Dict[str, Any]] = None
 
-        # Reset keys for backtesting
+        config['dry_run'] = True
-        remove_credentials(self.config)
         self.strategylist: List[IStrategy] = []
         self.all_results: Dict[str, Dict] = {}
 

freqtrade/optimize/edge_cli.py

@@ -7,7 +7,7 @@ import logging
 from typing import Any, Dict
 
 from freqtrade import constants
-from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
+from freqtrade.configuration import TimeRange, validate_config_consistency
 from freqtrade.edge import Edge
 from freqtrade.optimize.optimize_reports import generate_edge_table
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -28,8 +28,8 @@ class EdgeCli:
     def __init__(self, config: Dict[str, Any]) -> None:
         self.config = config
 
-        # Reset keys for edge
+        # Ensure using dry-run
-        remove_credentials(self.config)
+        self.config['dry_run'] = True
         self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT
         self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
         self.strategy = StrategyResolver.load_strategy(self.config)

freqtrade/plugins/pairlist/VolumePairList.py

@@ -123,7 +123,7 @@ class VolumePairList(IPairList):
             filtered_tickers = [
                 v for k, v in tickers.items()
                 if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
-                    and v[self._sort_key] is not None)]
+                    and (self._use_range or v[self._sort_key] is not None))]
             pairlist = [s['symbol'] for s in filtered_tickers]
 
             pairlist = self.filter_pairlist(pairlist, tickers)

requirements-hyperopt.txt

@@ -8,4 +8,4 @@ scikit-optimize==0.8.1
 filelock==3.0.12
 joblib==1.0.1
 psutil==5.8.0
-progressbar2==3.53.1
+progressbar2==3.53.2

requirements.txt

@@ -1,7 +1,7 @@
 numpy==1.21.2
-pandas==1.3.2
+pandas==1.3.3
 
-ccxt==1.55.83
+ccxt==1.56.30
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==3.4.8
 aiohttp==3.7.4.post0

tests/commands/test_commands.py

@@ -32,8 +32,6 @@ def test_setup_utils_configuration():
     config = setup_utils_configuration(get_args(args), RunMode.OTHER)
     assert "exchange" in config
     assert config['dry_run'] is True
-    assert config['exchange']['key'] == ''
-    assert config['exchange']['secret'] == ''
 
 
 def test_start_trading_fail(mocker, caplog):

tests/exchange/test_binance.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timezone
 from random import randint
 from unittest.mock import MagicMock
 
@@ -5,7 +6,7 @@ import ccxt
 import pytest
 
 from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException
-from tests.conftest import get_patched_exchange
+from tests.conftest import get_mock_coro, get_patched_exchange, log_has_re
 from tests.exchange.test_exchange import ccxt_exceptionhandlers
 
 
@@ -105,3 +106,35 @@ def test_stoploss_adjust_binance(mocker, default_conf):
     # Test with invalid order case
     order['type'] = 'stop_loss'
     assert not exchange.stoploss_adjust(1501, order)
+
+
+@pytest.mark.asyncio
+async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog):
+    ohlcv = [
+        [
+            int((datetime.now(timezone.utc).timestamp() - 1000) * 1000),
+            1,  # open
+            2,  # high
+            3,  # low
+            4,  # close
+            5,  # volume (in quote currency)
+        ]
+    ]
+
+    exchange = get_patched_exchange(mocker, default_conf, id='binance')
+    # Monkey-patch async function
+    exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv)
+
+    pair = 'ETH/BTC'
+    res = await exchange._async_get_historic_ohlcv(pair, "5m",
+                                                   1500000000000, is_new_pair=False)
+    # Call with very old timestamp - causes tons of requests
+    assert exchange._api_async.fetch_ohlcv.call_count > 400
+    # assert res == ohlcv
+    exchange._api_async.fetch_ohlcv.reset_mock()
+    res = await exchange._async_get_historic_ohlcv(pair, "5m", 1500000000000, is_new_pair=True)
+
+    # Called twice - one "init" call - and one to get the actual data.
+    assert exchange._api_async.fetch_ohlcv.call_count == 2
+    assert res == ohlcv
+    assert log_has_re(r"Candle-data for ETH/BTC available starting with .*", caplog)

tests/exchange/test_ccxt_compat.py

@@ -54,7 +54,7 @@ EXCHANGES = {
 def exchange_conf():
     config = get_default_conf((Path(__file__).parent / "testdata").resolve())
     config['exchange']['pair_whitelist'] = []
-    config['dry_run'] = False
+    # config['dry_run'] = False
     return config
 
 

tests/exchange/test_exchange.py

@@ -1,5 +1,6 @@
 import copy
 import logging
+from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from math import isclose
 from random import randint
@@ -14,7 +15,7 @@ from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOr
                                   OperationalException, PricingError, TemporaryError)
 from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken
 from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT,
-                                       calculate_backoff)
+                                       calculate_backoff, remove_credentials)
 from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_msecs,
                                          timeframe_to_next_date, timeframe_to_prev_date,
                                          timeframe_to_seconds)
@@ -78,6 +79,22 @@ def test_init(default_conf, mocker, caplog):
     assert log_has('Instance is running with dry_run enabled', caplog)
 
 
+def test_remove_credentials(default_conf, caplog) -> None:
+    conf = deepcopy(default_conf)
+    conf['dry_run'] = False
+    remove_credentials(conf)
+
+    assert conf['exchange']['key'] != ''
+    assert conf['exchange']['secret'] != ''
+
+    conf['dry_run'] = True
+    remove_credentials(conf)
+    assert conf['exchange']['key'] == ''
+    assert conf['exchange']['secret'] == ''
+    assert conf['exchange']['password'] == ''
+    assert conf['exchange']['uid'] == ''
+
+
 def test_init_ccxt_kwargs(default_conf, mocker, caplog):
     mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={}))
     mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
@@ -1551,6 +1568,32 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name):
     assert 'high' in ret.columns
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("exchange_name", EXCHANGES)
+async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name):
+    ohlcv = [
+        [
+            int((datetime.now(timezone.utc).timestamp() - 1000) * 1000),
+            1,  # open
+            2,  # high
+            3,  # low
+            4,  # close
+            5,  # volume (in quote currency)
+        ]
+    ]
+    exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
+    # Monkey-patch async function
+    exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv)
+
+    pair = 'ETH/USDT'
+    res = await exchange._async_get_historic_ohlcv(pair, "5m",
+                                                   1500000000000, is_new_pair=False)
+    # Call with very old timestamp - causes tons of requests
+    assert exchange._api_async.fetch_ohlcv.call_count > 200
+    assert res[0] == ohlcv[0]
+    assert log_has_re(r'Downloaded data for .* with length .*\.', caplog)
+
+
 def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None:
     ohlcv = [
         [

tests/test_configuration.py

@@ -11,8 +11,7 @@ import pytest
 from jsonschema import ValidationError
 
 from freqtrade.commands import Arguments
-from freqtrade.configuration import (Configuration, check_exchange, remove_credentials,
+from freqtrade.configuration import Configuration, check_exchange, validate_config_consistency
-                                     validate_config_consistency)
 from freqtrade.configuration.config_validation import validate_config_schema
 from freqtrade.configuration.deprecated_settings import (check_conflicting_settings,
                                                          process_deprecated_setting,
@@ -617,18 +616,6 @@ def test_check_exchange(default_conf, caplog) -> None:
         check_exchange(default_conf)
 
 
-def test_remove_credentials(default_conf, caplog) -> None:
-    conf = deepcopy(default_conf)
-    conf['dry_run'] = False
-    remove_credentials(conf)
-
-    assert conf['dry_run'] is True
-    assert conf['exchange']['key'] == ''
-    assert conf['exchange']['secret'] == ''
-    assert conf['exchange']['password'] == ''
-    assert conf['exchange']['uid'] == ''
-
-
 def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None:
     patched_configuration_load_config_file(mocker, default_conf)