Merge branch 'develop' into feat/short

docs/advanced-hyperopt.md

@@ -98,6 +98,38 @@ class MyAwesomeStrategy(IStrategy):
 !!! Note
     All overrides are optional and can be mixed/matched as necessary.
 
+### Overriding Base estimator
+
+You can define your own estimator for Hyperopt by implementing `generate_estimator()` in the Hyperopt subclass.
+
+```python
+class MyAwesomeStrategy(IStrategy):
+    class HyperOpt:
+        def generate_estimator():
+            return "RF"
+
+```
+
+Possible values are either one of "GP", "RF", "ET", "GBRT" (Details can be found in the [scikit-optimize documentation](https://scikit-optimize.github.io/)), or "an instance of a class that inherits from `RegressorMixin` (from sklearn) and where the `predict` method has an optional `return_std` argument, which returns `std(Y | x)` along with `E[Y | x]`".
+
+Some research will be necessary to find additional Regressors.
+
+Example for `ExtraTreesRegressor` ("ET") with additional parameters:
+
+```python
+class MyAwesomeStrategy(IStrategy):
+    class HyperOpt:
+        def generate_estimator():
+            from skopt.learning import ExtraTreesRegressor
+            # Corresponds to "ET" - but allows additional parameters.
+            return ExtraTreesRegressor(n_estimators=100)
+
+```
+
+!!! Note
+    While custom estimators can be provided, it's up to you as User to do research on possible parameters and analyze / understand which ones should be used.
+    If you're unsure about this, best use one of the Defaults (`"ET"` has proven to be the most versatile) without further parameters.
+
 ## Space options
 
 For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types:

docs/hyperopt.md

@@ -677,7 +677,7 @@ If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace f
 
 These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used.
 
-If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt file, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default.
+If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default.
 
 Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps).
 

freqtrade/configuration/PeriodicCache.py

@@ -0,0 +1,19 @@
+from datetime import datetime, timezone
+
+from cachetools.ttl import TTLCache
+
+
+class PeriodicCache(TTLCache):
+    """
+    Special cache that expires at "straight" times
+    A timer with ttl of 3600 (1h) will expire at every full hour (:00).
+    """
+
+    def __init__(self, maxsize, ttl, getsizeof=None):
+        def local_timer():
+            ts = datetime.now(timezone.utc).timestamp()
+            offset = (ts % ttl)
+            return ts - offset
+
+        # Init with smlight offset
+        super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof)

freqtrade/configuration/__init__.py

@@ -4,4 +4,5 @@ from freqtrade.configuration.check_exchange import check_exchange
 from freqtrade.configuration.config_setup import setup_utils_configuration
 from freqtrade.configuration.config_validation import validate_config_consistency
 from freqtrade.configuration.configuration import Configuration
+from freqtrade.configuration.PeriodicCache import PeriodicCache
 from freqtrade.configuration.timerange import TimeRange

freqtrade/optimize/hyperopt.py

@@ -45,7 +45,7 @@ progressbar.streams.wrap_stdout()
 logger = logging.getLogger(__name__)
 
 
-INITIAL_POINTS = 30
+INITIAL_POINTS = 5
 
 # Keep no more than SKOPT_MODEL_QUEUE_SIZE models
 # in the skopt model queue, to optimize memory consumption
@@ -241,7 +241,7 @@ class Hyperopt:
 
         if HyperoptTools.has_space(self.config, 'buy'):
             logger.debug("Hyperopt has 'buy' space")
-            self.buy_space = self.custom_hyperopt.indicator_space()
+            self.buy_space = self.custom_hyperopt.buy_indicator_space()
 
         if HyperoptTools.has_space(self.config, 'sell'):
             logger.debug("Hyperopt has 'sell' space")
@@ -365,10 +365,20 @@ class Hyperopt:
         }
 
     def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
+        estimator = self.custom_hyperopt.generate_estimator()
+
+        acq_optimizer = "sampling"
+        if isinstance(estimator, str):
+            if estimator not in ("GP", "RF", "ET", "GBRT"):
+                raise OperationalException(f"Estimator {estimator} not supported.")
+            else:
+                acq_optimizer = "auto"
+
+        logger.info(f"Using estimator {estimator}.")
         return Optimizer(
             dimensions,
-            base_estimator="ET",
-            acq_optimizer="auto",
+            base_estimator=estimator,
+            acq_optimizer=acq_optimizer,
             n_initial_points=INITIAL_POINTS,
             acq_optimizer_kwargs={'n_jobs': cpu_count},
             random_state=self.random_state,

freqtrade/optimize/hyperopt_auto.py

@@ -12,7 +12,7 @@ from freqtrade.exceptions import OperationalException
 with suppress(ImportError):
     from skopt.space import Dimension
 
-from freqtrade.optimize.hyperopt_interface import IHyperOpt
+from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt
 
 
 def _format_exception_message(space: str) -> str:
@@ -56,7 +56,7 @@ class HyperOptAuto(IHyperOpt):
         else:
             _format_exception_message(category)
 
-    def indicator_space(self) -> List['Dimension']:
+    def buy_indicator_space(self) -> List['Dimension']:
         return self._get_indicator_space('buy')
 
     def sell_indicator_space(self) -> List['Dimension']:
@@ -79,3 +79,6 @@ class HyperOptAuto(IHyperOpt):
 
     def trailing_space(self) -> List['Dimension']:
         return self._get_func('trailing_space')()
+
+    def generate_estimator(self) -> EstimatorType:
+        return self._get_func('generate_estimator')()

freqtrade/optimize/hyperopt_interface.py

@@ -5,8 +5,9 @@ This module defines the interface to apply for hyperopt
 import logging
 import math
 from abc import ABC
-from typing import Dict, List
+from typing import Dict, List, Union
 
+from sklearn.base import RegressorMixin
 from skopt.space import Categorical, Dimension, Integer
 
 from freqtrade.exchange import timeframe_to_minutes
@@ -17,6 +18,8 @@ from freqtrade.strategy import IStrategy
 
 logger = logging.getLogger(__name__)
 
+EstimatorType = Union[RegressorMixin, str]
+
 
 class IHyperOpt(ABC):
     """
@@ -37,6 +40,14 @@ class IHyperOpt(ABC):
         IHyperOpt.ticker_interval = str(config['timeframe'])  # DEPRECATED
         IHyperOpt.timeframe = str(config['timeframe'])
 
+    def generate_estimator(self) -> EstimatorType:
+        """
+        Return base_estimator.
+        Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class
+        inheriting from RegressorMixin (from sklearn).
+        """
+        return 'ET'
+
     def generate_roi_table(self, params: Dict) -> Dict[int, float]:
         """
         Create a ROI table.

freqtrade/plugins/pairlist/AgeFilter.py

@@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional
 import arrow
 from pandas import DataFrame
 
+from freqtrade.configuration import PeriodicCache
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import plural
 from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -18,14 +19,15 @@ logger = logging.getLogger(__name__)
 
 class AgeFilter(IPairList):
 
-    # Checked symbols cache (dictionary of ticker symbol => timestamp)
-    _symbolsChecked: Dict[str, int] = {}
-
     def __init__(self, exchange, pairlistmanager,
                  config: Dict[str, Any], pairlistconfig: Dict[str, Any],
                  pairlist_pos: int) -> None:
         super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
 
+        # Checked symbols cache (dictionary of ticker symbol => timestamp)
+        self._symbolsChecked: Dict[str, int] = {}
+        self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400)
+
         self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
         self._max_days_listed = pairlistconfig.get('max_days_listed', None)
 
@@ -69,9 +71,12 @@ class AgeFilter(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new allowlist
         """
-        needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked]
+        needed_pairs = [
+            (p, '1d') for p in pairlist
+            if p not in self._symbolsChecked and p not in self._symbolsCheckFailed]
         if not needed_pairs:
-            return pairlist
+            # Remove pairs that have been removed before
+            return [p for p in pairlist if p not in self._symbolsCheckFailed]
 
         since_days = -(
             self._max_days_listed if self._max_days_listed else self._min_days_listed
@@ -118,5 +123,6 @@ class AgeFilter(IPairList):
                     " or more than "
                     f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
                 ) if self._max_days_listed else ''), logger.info)
+                self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000
                 return False
         return False

freqtrade/strategy/interface.py

@@ -786,10 +786,11 @@ class IStrategy(ABC, HyperStrategyMixin):
         Does not run advise_buy or advise_sell!
         Used by optimize operations only, not during dry / live runs.
         Using .copy() to get a fresh copy of the dataframe for every strategy run.
+        Also copy on output to avoid PerformanceWarnings pandas 1.3.0 started to show.
         Has positive effects on memory usage for whatever reason - also when
         using only one strategy.
         """
-        return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair})
+        return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair}).copy()
                 for pair, pair_data in data.items()}
 
     def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

requirements-dev.txt

@@ -14,6 +14,8 @@ pytest-cov==2.12.1
 pytest-mock==3.6.1
 pytest-random-order==1.0.4
 isort==5.9.3
+# For datetime mocking
+time-machine==2.4.0
 
 # Convert jupyter notebooks to markdown documents
 nbconvert==6.1.0

tests/optimize/test_hyperopt.py

@@ -884,6 +884,10 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None:
     assert hyperopt.backtesting.strategy.buy_rsi.value != 35
     assert hyperopt.backtesting.strategy.sell_rsi.value != 74
 
+    hyperopt.custom_hyperopt.generate_estimator = lambda *args, **kwargs: 'ET1'
+    with pytest.raises(OperationalException, match="Estimator ET1 not supported."):
+        hyperopt.get_optimizer([], 2)
+
 
 def test_SKDecimal():
     space = SKDecimal(1, 2, decimals=2)

tests/plugins/test_pairlist.py

@@ -4,6 +4,7 @@ import time
 from unittest.mock import MagicMock, PropertyMock
 
 import pytest
+import time_machine
 
 from freqtrade.constants import AVAILABLE_PAIRLISTS
 from freqtrade.exceptions import OperationalException
@@ -815,32 +816,63 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick
 
 
 def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history):
-    ohlcv_data = {
-        ('ETH/BTC', '1d'): ohlcv_history,
-        ('TKN/BTC', '1d'): ohlcv_history,
-        ('LTC/BTC', '1d'): ohlcv_history,
-    }
-    mocker.patch.multiple('freqtrade.exchange.Exchange',
-                          markets=PropertyMock(return_value=markets),
-                          exchange_has=MagicMock(return_value=True),
-                          get_tickers=tickers
-                          )
-    mocker.patch.multiple(
-        'freqtrade.exchange.Exchange',
-        refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
-    )
+    with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
+        ohlcv_data = {
+            ('ETH/BTC', '1d'): ohlcv_history,
+            ('TKN/BTC', '1d'): ohlcv_history,
+            ('LTC/BTC', '1d'): ohlcv_history,
+        }
+        mocker.patch.multiple(
+            'freqtrade.exchange.Exchange',
+            markets=PropertyMock(return_value=markets),
+            exchange_has=MagicMock(return_value=True),
+            get_tickers=tickers,
+            refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
+        )
 
-    freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter)
-    assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0
-    freqtrade.pairlists.refresh_pairlist()
-    assert len(freqtrade.pairlists.whitelist) == 3
-    assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0
+        freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter)
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0
+        freqtrade.pairlists.refresh_pairlist()
+        assert len(freqtrade.pairlists.whitelist) == 3
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0
 
-    previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count
-    freqtrade.pairlists.refresh_pairlist()
-    assert len(freqtrade.pairlists.whitelist) == 3
-    # Called once for XRP/BTC
-    assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1
+        freqtrade.pairlists.refresh_pairlist()
+        assert len(freqtrade.pairlists.whitelist) == 3
+        # Call to XRP/BTC cached
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2
+
+        ohlcv_data = {
+            ('ETH/BTC', '1d'): ohlcv_history,
+            ('TKN/BTC', '1d'): ohlcv_history,
+            ('LTC/BTC', '1d'): ohlcv_history,
+            ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]],
+        }
+        mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
+        freqtrade.pairlists.refresh_pairlist()
+        assert len(freqtrade.pairlists.whitelist) == 3
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
+
+        # Move to next day
+        t.move_to("2021-09-02 01:00:00 +00:00")
+        mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
+        freqtrade.pairlists.refresh_pairlist()
+        assert len(freqtrade.pairlists.whitelist) == 3
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
+
+        # Move another day with fresh mocks (now the pair is old enough)
+        t.move_to("2021-09-03 01:00:00 +00:00")
+        # Called once for XRP/BTC
+        ohlcv_data = {
+            ('ETH/BTC', '1d'): ohlcv_history,
+            ('TKN/BTC', '1d'): ohlcv_history,
+            ('LTC/BTC', '1d'): ohlcv_history,
+            ('XRP/BTC', '1d'): ohlcv_history,
+        }
+        mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
+        freqtrade.pairlists.refresh_pairlist()
+        assert len(freqtrade.pairlists.whitelist) == 4
+        # Called once (only for XRP/BTC)
+        assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
 
 
 def test_OffsetFilter_error(mocker, whitelist_conf) -> None:

tests/test_periodiccache.py

@@ -0,0 +1,32 @@
+import time_machine
+
+from freqtrade.configuration import PeriodicCache
+
+
+def test_ttl_cache():
+
+    with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
+
+        cache = PeriodicCache(5, ttl=60)
+        cache1h = PeriodicCache(5, ttl=3600)
+
+        assert cache.timer() == 1630472400.0
+        cache['a'] = 1235
+        cache1h['a'] = 555123
+        assert 'a' in cache
+        assert 'a' in cache1h
+
+        t.move_to("2021-09-01 05:00:59 +00:00")
+        assert 'a' in cache
+        assert 'a' in cache1h
+
+        # Cache expired
+        t.move_to("2021-09-01 05:01:00 +00:00")
+        assert 'a' not in cache
+        assert 'a' in cache1h
+
+        t.move_to("2021-09-01 05:59:59 +00:00")
+        assert 'a' in cache1h
+
+        t.move_to("2021-09-01 06:00:00 +00:00")
+        assert 'a' not in cache1h