Merge branch 'feat/short' into lev-exchange

This commit is contained in:
Sam Germain 2021-09-19 01:57:30 -06:00
commit 2b6d134294
13 changed files with 191 additions and 38 deletions

View File

@ -98,6 +98,38 @@ class MyAwesomeStrategy(IStrategy):
!!! Note !!! Note
All overrides are optional and can be mixed/matched as necessary. All overrides are optional and can be mixed/matched as necessary.
### Overriding Base estimator
You can define your own estimator for Hyperopt by implementing `generate_estimator()` in the Hyperopt subclass.
```python
class MyAwesomeStrategy(IStrategy):
class HyperOpt:
def generate_estimator():
return "RF"
```
Possible values are either one of "GP", "RF", "ET", "GBRT" (Details can be found in the [scikit-optimize documentation](https://scikit-optimize.github.io/)), or "an instance of a class that inherits from `RegressorMixin` (from sklearn) and where the `predict` method has an optional `return_std` argument, which returns `std(Y | x)` along with `E[Y | x]`".
Some research will be necessary to find additional Regressors.
Example for `ExtraTreesRegressor` ("ET") with additional parameters:
```python
class MyAwesomeStrategy(IStrategy):
class HyperOpt:
def generate_estimator():
from skopt.learning import ExtraTreesRegressor
# Corresponds to "ET" - but allows additional parameters.
return ExtraTreesRegressor(n_estimators=100)
```
!!! Note
While custom estimators can be provided, it's up to you as User to do research on possible parameters and analyze / understand which ones should be used.
If you're unsure about this, best use one of the Defaults (`"ET"` has proven to be the most versatile) without further parameters.
## Space options ## Space options
For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types: For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types:

View File

@ -677,7 +677,7 @@ If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace f
These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used. These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used.
If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt file, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default. If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default.
Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps). Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps).

View File

@ -0,0 +1,19 @@
from datetime import datetime, timezone
from cachetools.ttl import TTLCache
class PeriodicCache(TTLCache):
"""
Special cache that expires at "straight" times
A timer with ttl of 3600 (1h) will expire at every full hour (:00).
"""
def __init__(self, maxsize, ttl, getsizeof=None):
def local_timer():
ts = datetime.now(timezone.utc).timestamp()
offset = (ts % ttl)
return ts - offset
# Init with smlight offset
super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof)

View File

@ -4,4 +4,5 @@ from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_setup import setup_utils_configuration
from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.configuration.config_validation import validate_config_consistency
from freqtrade.configuration.configuration import Configuration from freqtrade.configuration.configuration import Configuration
from freqtrade.configuration.PeriodicCache import PeriodicCache
from freqtrade.configuration.timerange import TimeRange from freqtrade.configuration.timerange import TimeRange

View File

@ -45,7 +45,7 @@ progressbar.streams.wrap_stdout()
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
INITIAL_POINTS = 30 INITIAL_POINTS = 5
# Keep no more than SKOPT_MODEL_QUEUE_SIZE models # Keep no more than SKOPT_MODEL_QUEUE_SIZE models
# in the skopt model queue, to optimize memory consumption # in the skopt model queue, to optimize memory consumption
@ -241,7 +241,7 @@ class Hyperopt:
if HyperoptTools.has_space(self.config, 'buy'): if HyperoptTools.has_space(self.config, 'buy'):
logger.debug("Hyperopt has 'buy' space") logger.debug("Hyperopt has 'buy' space")
self.buy_space = self.custom_hyperopt.indicator_space() self.buy_space = self.custom_hyperopt.buy_indicator_space()
if HyperoptTools.has_space(self.config, 'sell'): if HyperoptTools.has_space(self.config, 'sell'):
logger.debug("Hyperopt has 'sell' space") logger.debug("Hyperopt has 'sell' space")
@ -365,10 +365,20 @@ class Hyperopt:
} }
def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
estimator = self.custom_hyperopt.generate_estimator()
acq_optimizer = "sampling"
if isinstance(estimator, str):
if estimator not in ("GP", "RF", "ET", "GBRT"):
raise OperationalException(f"Estimator {estimator} not supported.")
else:
acq_optimizer = "auto"
logger.info(f"Using estimator {estimator}.")
return Optimizer( return Optimizer(
dimensions, dimensions,
base_estimator="ET", base_estimator=estimator,
acq_optimizer="auto", acq_optimizer=acq_optimizer,
n_initial_points=INITIAL_POINTS, n_initial_points=INITIAL_POINTS,
acq_optimizer_kwargs={'n_jobs': cpu_count}, acq_optimizer_kwargs={'n_jobs': cpu_count},
random_state=self.random_state, random_state=self.random_state,

View File

@ -12,7 +12,7 @@ from freqtrade.exceptions import OperationalException
with suppress(ImportError): with suppress(ImportError):
from skopt.space import Dimension from skopt.space import Dimension
from freqtrade.optimize.hyperopt_interface import IHyperOpt from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt
def _format_exception_message(space: str) -> str: def _format_exception_message(space: str) -> str:
@ -56,7 +56,7 @@ class HyperOptAuto(IHyperOpt):
else: else:
_format_exception_message(category) _format_exception_message(category)
def indicator_space(self) -> List['Dimension']: def buy_indicator_space(self) -> List['Dimension']:
return self._get_indicator_space('buy') return self._get_indicator_space('buy')
def sell_indicator_space(self) -> List['Dimension']: def sell_indicator_space(self) -> List['Dimension']:
@ -79,3 +79,6 @@ class HyperOptAuto(IHyperOpt):
def trailing_space(self) -> List['Dimension']: def trailing_space(self) -> List['Dimension']:
return self._get_func('trailing_space')() return self._get_func('trailing_space')()
def generate_estimator(self) -> EstimatorType:
return self._get_func('generate_estimator')()

View File

@ -5,8 +5,9 @@ This module defines the interface to apply for hyperopt
import logging import logging
import math import math
from abc import ABC from abc import ABC
from typing import Dict, List from typing import Dict, List, Union
from sklearn.base import RegressorMixin
from skopt.space import Categorical, Dimension, Integer from skopt.space import Categorical, Dimension, Integer
from freqtrade.exchange import timeframe_to_minutes from freqtrade.exchange import timeframe_to_minutes
@ -17,6 +18,8 @@ from freqtrade.strategy import IStrategy
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
EstimatorType = Union[RegressorMixin, str]
class IHyperOpt(ABC): class IHyperOpt(ABC):
""" """
@ -37,6 +40,14 @@ class IHyperOpt(ABC):
IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED
IHyperOpt.timeframe = str(config['timeframe']) IHyperOpt.timeframe = str(config['timeframe'])
def generate_estimator(self) -> EstimatorType:
"""
Return base_estimator.
Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class
inheriting from RegressorMixin (from sklearn).
"""
return 'ET'
def generate_roi_table(self, params: Dict) -> Dict[int, float]: def generate_roi_table(self, params: Dict) -> Dict[int, float]:
""" """
Create a ROI table. Create a ROI table.

View File

@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional
import arrow import arrow
from pandas import DataFrame from pandas import DataFrame
from freqtrade.configuration import PeriodicCache
from freqtrade.exceptions import OperationalException from freqtrade.exceptions import OperationalException
from freqtrade.misc import plural from freqtrade.misc import plural
from freqtrade.plugins.pairlist.IPairList import IPairList from freqtrade.plugins.pairlist.IPairList import IPairList
@ -18,14 +19,15 @@ logger = logging.getLogger(__name__)
class AgeFilter(IPairList): class AgeFilter(IPairList):
# Checked symbols cache (dictionary of ticker symbol => timestamp)
_symbolsChecked: Dict[str, int] = {}
def __init__(self, exchange, pairlistmanager, def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any], config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None: pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
# Checked symbols cache (dictionary of ticker symbol => timestamp)
self._symbolsChecked: Dict[str, int] = {}
self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400)
self._min_days_listed = pairlistconfig.get('min_days_listed', 10) self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
self._max_days_listed = pairlistconfig.get('max_days_listed', None) self._max_days_listed = pairlistconfig.get('max_days_listed', None)
@ -69,9 +71,12 @@ class AgeFilter(IPairList):
:param tickers: Tickers (from exchange.get_tickers()). May be cached. :param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new allowlist :return: new allowlist
""" """
needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked] needed_pairs = [
(p, '1d') for p in pairlist
if p not in self._symbolsChecked and p not in self._symbolsCheckFailed]
if not needed_pairs: if not needed_pairs:
return pairlist # Remove pairs that have been removed before
return [p for p in pairlist if p not in self._symbolsCheckFailed]
since_days = -( since_days = -(
self._max_days_listed if self._max_days_listed else self._min_days_listed self._max_days_listed if self._max_days_listed else self._min_days_listed
@ -118,5 +123,6 @@ class AgeFilter(IPairList):
" or more than " " or more than "
f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
) if self._max_days_listed else ''), logger.info) ) if self._max_days_listed else ''), logger.info)
self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000
return False return False
return False return False

View File

@ -786,10 +786,11 @@ class IStrategy(ABC, HyperStrategyMixin):
Does not run advise_buy or advise_sell! Does not run advise_buy or advise_sell!
Used by optimize operations only, not during dry / live runs. Used by optimize operations only, not during dry / live runs.
Using .copy() to get a fresh copy of the dataframe for every strategy run. Using .copy() to get a fresh copy of the dataframe for every strategy run.
Also copy on output to avoid PerformanceWarnings pandas 1.3.0 started to show.
Has positive effects on memory usage for whatever reason - also when Has positive effects on memory usage for whatever reason - also when
using only one strategy. using only one strategy.
""" """
return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair}) return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair}).copy()
for pair, pair_data in data.items()} for pair, pair_data in data.items()}
def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

View File

@ -14,6 +14,8 @@ pytest-cov==2.12.1
pytest-mock==3.6.1 pytest-mock==3.6.1
pytest-random-order==1.0.4 pytest-random-order==1.0.4
isort==5.9.3 isort==5.9.3
# For datetime mocking
time-machine==2.4.0
# Convert jupyter notebooks to markdown documents # Convert jupyter notebooks to markdown documents
nbconvert==6.1.0 nbconvert==6.1.0

View File

@ -884,6 +884,10 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None:
assert hyperopt.backtesting.strategy.buy_rsi.value != 35 assert hyperopt.backtesting.strategy.buy_rsi.value != 35
assert hyperopt.backtesting.strategy.sell_rsi.value != 74 assert hyperopt.backtesting.strategy.sell_rsi.value != 74
hyperopt.custom_hyperopt.generate_estimator = lambda *args, **kwargs: 'ET1'
with pytest.raises(OperationalException, match="Estimator ET1 not supported."):
hyperopt.get_optimizer([], 2)
def test_SKDecimal(): def test_SKDecimal():
space = SKDecimal(1, 2, decimals=2) space = SKDecimal(1, 2, decimals=2)

View File

@ -4,6 +4,7 @@ import time
from unittest.mock import MagicMock, PropertyMock from unittest.mock import MagicMock, PropertyMock
import pytest import pytest
import time_machine
from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.constants import AVAILABLE_PAIRLISTS
from freqtrade.exceptions import OperationalException from freqtrade.exceptions import OperationalException
@ -815,32 +816,63 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick
def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history): def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history):
ohlcv_data = { with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
('ETH/BTC', '1d'): ohlcv_history, ohlcv_data = {
('TKN/BTC', '1d'): ohlcv_history, ('ETH/BTC', '1d'): ohlcv_history,
('LTC/BTC', '1d'): ohlcv_history, ('TKN/BTC', '1d'): ohlcv_history,
} ('LTC/BTC', '1d'): ohlcv_history,
mocker.patch.multiple('freqtrade.exchange.Exchange', }
markets=PropertyMock(return_value=markets), mocker.patch.multiple(
exchange_has=MagicMock(return_value=True), 'freqtrade.exchange.Exchange',
get_tickers=tickers markets=PropertyMock(return_value=markets),
) exchange_has=MagicMock(return_value=True),
mocker.patch.multiple( get_tickers=tickers,
'freqtrade.exchange.Exchange', refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), )
)
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter)
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0
freqtrade.pairlists.refresh_pairlist() freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 3 assert len(freqtrade.pairlists.whitelist) == 3
assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0
previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count freqtrade.pairlists.refresh_pairlist()
freqtrade.pairlists.refresh_pairlist() assert len(freqtrade.pairlists.whitelist) == 3
assert len(freqtrade.pairlists.whitelist) == 3 # Call to XRP/BTC cached
# Called once for XRP/BTC assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1
ohlcv_data = {
('ETH/BTC', '1d'): ohlcv_history,
('TKN/BTC', '1d'): ohlcv_history,
('LTC/BTC', '1d'): ohlcv_history,
('XRP/BTC', '1d'): ohlcv_history.iloc[[0]],
}
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 3
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
# Move to next day
t.move_to("2021-09-02 01:00:00 +00:00")
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 3
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
# Move another day with fresh mocks (now the pair is old enough)
t.move_to("2021-09-03 01:00:00 +00:00")
# Called once for XRP/BTC
ohlcv_data = {
('ETH/BTC', '1d'): ohlcv_history,
('TKN/BTC', '1d'): ohlcv_history,
('LTC/BTC', '1d'): ohlcv_history,
('XRP/BTC', '1d'): ohlcv_history,
}
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 4
# Called once (only for XRP/BTC)
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
def test_OffsetFilter_error(mocker, whitelist_conf) -> None: def test_OffsetFilter_error(mocker, whitelist_conf) -> None:

View File

@ -0,0 +1,32 @@
import time_machine
from freqtrade.configuration import PeriodicCache
def test_ttl_cache():
with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
cache = PeriodicCache(5, ttl=60)
cache1h = PeriodicCache(5, ttl=3600)
assert cache.timer() == 1630472400.0
cache['a'] = 1235
cache1h['a'] = 555123
assert 'a' in cache
assert 'a' in cache1h
t.move_to("2021-09-01 05:00:59 +00:00")
assert 'a' in cache
assert 'a' in cache1h
# Cache expired
t.move_to("2021-09-01 05:01:00 +00:00")
assert 'a' not in cache
assert 'a' in cache1h
t.move_to("2021-09-01 05:59:59 +00:00")
assert 'a' in cache1h
t.move_to("2021-09-01 06:00:00 +00:00")
assert 'a' not in cache1h