Merge branch 'develop' into feat/short

freqtrade/configuration/PeriodicCache.py

@@ -0,0 +1,19 @@
+from datetime import datetime, timezone
+
+from cachetools.ttl import TTLCache
+
+
+class PeriodicCache(TTLCache):
+    """
+    Special cache that expires at "straight" times
+    A timer with ttl of 3600 (1h) will expire at every full hour (:00).
+    """
+
+    def __init__(self, maxsize, ttl, getsizeof=None):
+        def local_timer():
+            ts = datetime.now(timezone.utc).timestamp()
+            offset = (ts % ttl)
+            return ts - offset
+
+        # Init with smlight offset
+        super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof)

freqtrade/configuration/__init__.py

@@ -4,4 +4,5 @@ from freqtrade.configuration.check_exchange import check_exchange
 from freqtrade.configuration.config_setup import setup_utils_configuration
 from freqtrade.configuration.config_validation import validate_config_consistency
 from freqtrade.configuration.configuration import Configuration
+from freqtrade.configuration.PeriodicCache import PeriodicCache
 from freqtrade.configuration.timerange import TimeRange

freqtrade/optimize/hyperopt.py

@@ -45,7 +45,7 @@ progressbar.streams.wrap_stdout()
 logger = logging.getLogger(__name__)
 
 
-INITIAL_POINTS = 30
+INITIAL_POINTS = 5
 
 # Keep no more than SKOPT_MODEL_QUEUE_SIZE models
 # in the skopt model queue, to optimize memory consumption
@@ -241,7 +241,7 @@ class Hyperopt:
 
         if HyperoptTools.has_space(self.config, 'buy'):
             logger.debug("Hyperopt has 'buy' space")
-            self.buy_space = self.custom_hyperopt.indicator_space()
+            self.buy_space = self.custom_hyperopt.buy_indicator_space()
 
         if HyperoptTools.has_space(self.config, 'sell'):
             logger.debug("Hyperopt has 'sell' space")
@@ -365,10 +365,20 @@ class Hyperopt:
         }
 
     def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
+        estimator = self.custom_hyperopt.generate_estimator()
+
+        acq_optimizer = "sampling"
+        if isinstance(estimator, str):
+            if estimator not in ("GP", "RF", "ET", "GBRT"):
+                raise OperationalException(f"Estimator {estimator} not supported.")
+            else:
+                acq_optimizer = "auto"
+
+        logger.info(f"Using estimator {estimator}.")
         return Optimizer(
             dimensions,
-            base_estimator="ET",
-            acq_optimizer="auto",
+            base_estimator=estimator,
+            acq_optimizer=acq_optimizer,
             n_initial_points=INITIAL_POINTS,
             acq_optimizer_kwargs={'n_jobs': cpu_count},
             random_state=self.random_state,

freqtrade/optimize/hyperopt_auto.py

@@ -12,7 +12,7 @@ from freqtrade.exceptions import OperationalException
 with suppress(ImportError):
     from skopt.space import Dimension
 
-from freqtrade.optimize.hyperopt_interface import IHyperOpt
+from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt
 
 
 def _format_exception_message(space: str) -> str:
@@ -56,7 +56,7 @@ class HyperOptAuto(IHyperOpt):
         else:
             _format_exception_message(category)
 
-    def indicator_space(self) -> List['Dimension']:
+    def buy_indicator_space(self) -> List['Dimension']:
         return self._get_indicator_space('buy')
 
     def sell_indicator_space(self) -> List['Dimension']:
@@ -79,3 +79,6 @@ class HyperOptAuto(IHyperOpt):
 
     def trailing_space(self) -> List['Dimension']:
         return self._get_func('trailing_space')()
+
+    def generate_estimator(self) -> EstimatorType:
+        return self._get_func('generate_estimator')()

freqtrade/optimize/hyperopt_interface.py

@@ -5,8 +5,9 @@ This module defines the interface to apply for hyperopt
 import logging
 import math
 from abc import ABC
-from typing import Dict, List
+from typing import Dict, List, Union
 
+from sklearn.base import RegressorMixin
 from skopt.space import Categorical, Dimension, Integer
 
 from freqtrade.exchange import timeframe_to_minutes
@@ -17,6 +18,8 @@ from freqtrade.strategy import IStrategy
 
 logger = logging.getLogger(__name__)
 
+EstimatorType = Union[RegressorMixin, str]
+
 
 class IHyperOpt(ABC):
     """
@@ -37,6 +40,14 @@ class IHyperOpt(ABC):
         IHyperOpt.ticker_interval = str(config['timeframe'])  # DEPRECATED
         IHyperOpt.timeframe = str(config['timeframe'])
 
+    def generate_estimator(self) -> EstimatorType:
+        """
+        Return base_estimator.
+        Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class
+        inheriting from RegressorMixin (from sklearn).
+        """
+        return 'ET'
+
     def generate_roi_table(self, params: Dict) -> Dict[int, float]:
         """
         Create a ROI table.

freqtrade/plugins/pairlist/AgeFilter.py

@@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional
 import arrow
 from pandas import DataFrame
 
+from freqtrade.configuration import PeriodicCache
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import plural
 from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -18,14 +19,15 @@ logger = logging.getLogger(__name__)
 
 class AgeFilter(IPairList):
 
-    # Checked symbols cache (dictionary of ticker symbol => timestamp)
-    _symbolsChecked: Dict[str, int] = {}
-
     def __init__(self, exchange, pairlistmanager,
                  config: Dict[str, Any], pairlistconfig: Dict[str, Any],
                  pairlist_pos: int) -> None:
         super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
 
+        # Checked symbols cache (dictionary of ticker symbol => timestamp)
+        self._symbolsChecked: Dict[str, int] = {}
+        self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400)
+
         self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
         self._max_days_listed = pairlistconfig.get('max_days_listed', None)
 
@@ -69,9 +71,12 @@ class AgeFilter(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new allowlist
         """
-        needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked]
+        needed_pairs = [
+            (p, '1d') for p in pairlist
+            if p not in self._symbolsChecked and p not in self._symbolsCheckFailed]
         if not needed_pairs:
-            return pairlist
+            # Remove pairs that have been removed before
+            return [p for p in pairlist if p not in self._symbolsCheckFailed]
 
         since_days = -(
             self._max_days_listed if self._max_days_listed else self._min_days_listed
@@ -118,5 +123,6 @@ class AgeFilter(IPairList):
                     " or more than "
                     f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
                 ) if self._max_days_listed else ''), logger.info)
+                self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000
                 return False
         return False

freqtrade/strategy/interface.py

@@ -786,10 +786,11 @@ class IStrategy(ABC, HyperStrategyMixin):
         Does not run advise_buy or advise_sell!
         Used by optimize operations only, not during dry / live runs.
         Using .copy() to get a fresh copy of the dataframe for every strategy run.
+        Also copy on output to avoid PerformanceWarnings pandas 1.3.0 started to show.
         Has positive effects on memory usage for whatever reason - also when
         using only one strategy.
         """
-        return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair})
+        return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair}).copy()
                 for pair, pair_data in data.items()}
 
     def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: