adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into feat/short

Browse Source
This commit is contained in:
Matthias 2021-09-17 11:16:37 +02:00
commit d680fdf33a
13 changed files with 191 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
docs/advanced-hyperopt.md
View File

@ -98,6 +98,38 @@ class MyAwesomeStrategy(IStrategy):
!!! Note
All overrides are optional and can be mixed/matched as necessary.
### Overriding Base estimator
You can define your own estimator for Hyperopt by implementing `generate_estimator()` in the Hyperopt subclass.
```python
class MyAwesomeStrategy(IStrategy):
class HyperOpt:
def generate_estimator():
return "RF"
```
Possible values are either one of "GP", "RF", "ET", "GBRT" (Details can be found in the [scikit-optimize documentation](https://scikit-optimize.github.io/)), or "an instance of a class that inherits from `RegressorMixin` (from sklearn) and where the `predict` method has an optional `return_std` argument, which returns `std(Y | x)` along with `E[Y | x]`".
Some research will be necessary to find additional Regressors.
Example for `ExtraTreesRegressor` ("ET") with additional parameters:
```python
class MyAwesomeStrategy(IStrategy):
class HyperOpt:
def generate_estimator():
from skopt.learning import ExtraTreesRegressor
# Corresponds to "ET" - but allows additional parameters.
return ExtraTreesRegressor(n_estimators=100)
```
!!! Note
While custom estimators can be provided, it's up to you as User to do research on possible parameters and analyze / understand which ones should be used.
If you're unsure about this, best use one of the Defaults (`"ET"` has proven to be the most versatile) without further parameters.
## Space options
For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types:

2
docs/hyperopt.md
View File

@ -677,7 +677,7 @@ If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace f
These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used.
If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt file, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default.
If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default.
Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps).

19
freqtrade/configuration/PeriodicCache.py Normal file
View File

@ -0,0 +1,19 @@
from datetime import datetime, timezone
from cachetools.ttl import TTLCache
class PeriodicCache(TTLCache):
"""
Special cache that expires at "straight" times
A timer with ttl of 3600 (1h) will expire at every full hour (:00).
"""
def __init__(self, maxsize, ttl, getsizeof=None):
def local_timer():
ts = datetime.now(timezone.utc).timestamp()
offset = (ts % ttl)
return ts - offset
# Init with smlight offset
super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof)

1
freqtrade/configuration/__init__.py
View File

@ -4,4 +4,5 @@ from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.config_setup import setup_utils_configuration
from freqtrade.configuration.config_validation import validate_config_consistency
from freqtrade.configuration.configuration import Configuration
from freqtrade.configuration.PeriodicCache import PeriodicCache
from freqtrade.configuration.timerange import TimeRange

18
freqtrade/optimize/hyperopt.py
View File

@ -45,7 +45,7 @@ progressbar.streams.wrap_stdout()
logger = logging.getLogger(__name__)
INITIAL_POINTS = 30
INITIAL_POINTS = 5
# Keep no more than SKOPT_MODEL_QUEUE_SIZE models
# in the skopt model queue, to optimize memory consumption
@ -241,7 +241,7 @@ class Hyperopt:
if HyperoptTools.has_space(self.config, 'buy'):
logger.debug("Hyperopt has 'buy' space")
self.buy_space = self.custom_hyperopt.indicator_space()
self.buy_space = self.custom_hyperopt.buy_indicator_space()
if HyperoptTools.has_space(self.config, 'sell'):
logger.debug("Hyperopt has 'sell' space")
@ -365,10 +365,20 @@ class Hyperopt:
}
def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
estimator = self.custom_hyperopt.generate_estimator()
acq_optimizer = "sampling"
if isinstance(estimator, str):
if estimator not in ("GP", "RF", "ET", "GBRT"):
raise OperationalException(f"Estimator {estimator} not supported.")
else:
acq_optimizer = "auto"
logger.info(f"Using estimator {estimator}.")
return Optimizer(
dimensions,
base_estimator="ET",
acq_optimizer="auto",
base_estimator=estimator,
acq_optimizer=acq_optimizer,
n_initial_points=INITIAL_POINTS,
acq_optimizer_kwargs={'n_jobs': cpu_count},
random_state=self.random_state,

7
freqtrade/optimize/hyperopt_auto.py
View File

@ -12,7 +12,7 @@ from freqtrade.exceptions import OperationalException
with suppress(ImportError):
from skopt.space import Dimension
from freqtrade.optimize.hyperopt_interface import IHyperOpt
from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt
def _format_exception_message(space: str) -> str:
@ -56,7 +56,7 @@ class HyperOptAuto(IHyperOpt):
else:
_format_exception_message(category)
def indicator_space(self) -> List['Dimension']:
def buy_indicator_space(self) -> List['Dimension']:
return self._get_indicator_space('buy')
def sell_indicator_space(self) -> List['Dimension']:
@ -79,3 +79,6 @@ class HyperOptAuto(IHyperOpt):
def trailing_space(self) -> List['Dimension']:
return self._get_func('trailing_space')()
def generate_estimator(self) -> EstimatorType:
return self._get_func('generate_estimator')()

13
freqtrade/optimize/hyperopt_interface.py
View File

@ -5,8 +5,9 @@ This module defines the interface to apply for hyperopt
import logging
import math
from abc import ABC
from typing import Dict, List
from typing import Dict, List, Union
from sklearn.base import RegressorMixin
from skopt.space import Categorical, Dimension, Integer
from freqtrade.exchange import timeframe_to_minutes
@ -17,6 +18,8 @@ from freqtrade.strategy import IStrategy
logger = logging.getLogger(__name__)
EstimatorType = Union[RegressorMixin, str]
class IHyperOpt(ABC):
"""
@ -37,6 +40,14 @@ class IHyperOpt(ABC):
IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED
IHyperOpt.timeframe = str(config['timeframe'])
def generate_estimator(self) -> EstimatorType:
"""
Return base_estimator.
Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class
inheriting from RegressorMixin (from sklearn).
"""
return 'ET'
def generate_roi_table(self, params: Dict) -> Dict[int, float]:
"""
Create a ROI table.

16
freqtrade/plugins/pairlist/AgeFilter.py
View File

@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional
import arrow
from pandas import DataFrame
from freqtrade.configuration import PeriodicCache
from freqtrade.exceptions import OperationalException
from freqtrade.misc import plural
from freqtrade.plugins.pairlist.IPairList import IPairList
@ -18,14 +19,15 @@ logger = logging.getLogger(__name__)
class AgeFilter(IPairList):
# Checked symbols cache (dictionary of ticker symbol => timestamp)
_symbolsChecked: Dict[str, int] = {}
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
# Checked symbols cache (dictionary of ticker symbol => timestamp)
self._symbolsChecked: Dict[str, int] = {}
self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400)
self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
self._max_days_listed = pairlistconfig.get('max_days_listed', None)
@ -69,9 +71,12 @@ class AgeFilter(IPairList):
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new allowlist
"""
needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked]
needed_pairs = [
(p, '1d') for p in pairlist
if p not in self._symbolsChecked and p not in self._symbolsCheckFailed]
if not needed_pairs:
return pairlist
# Remove pairs that have been removed before
return [p for p in pairlist if p not in self._symbolsCheckFailed]
since_days = -(
self._max_days_listed if self._max_days_listed else self._min_days_listed
@ -118,5 +123,6 @@ class AgeFilter(IPairList):
" or more than "
f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
) if self._max_days_listed else ''), logger.info)
self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000
return False
return False

3
freqtrade/strategy/interface.py
View File

@ -786,10 +786,11 @@ class IStrategy(ABC, HyperStrategyMixin):
Does not run advise_buy or advise_sell!
Used by optimize operations only, not during dry / live runs.
Using .copy() to get a fresh copy of the dataframe for every strategy run.
Also copy on output to avoid PerformanceWarnings pandas 1.3.0 started to show.
Has positive effects on memory usage for whatever reason - also when
using only one strategy.
"""
return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair})
return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair}).copy()
for pair, pair_data in data.items()}
def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

2
requirements-dev.txt
View File

@ -14,6 +14,8 @@ pytest-cov==2.12.1
pytest-mock==3.6.1
pytest-random-order==1.0.4
isort==5.9.3
# For datetime mocking
time-machine==2.4.0
# Convert jupyter notebooks to markdown documents
nbconvert==6.1.0

4
tests/optimize/test_hyperopt.py
View File

@ -884,6 +884,10 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None:
assert hyperopt.backtesting.strategy.buy_rsi.value != 35
assert hyperopt.backtesting.strategy.sell_rsi.value != 74
hyperopt.custom_hyperopt.generate_estimator = lambda *args, **kwargs: 'ET1'
with pytest.raises(OperationalException, match="Estimator ET1 not supported."):
hyperopt.get_optimizer([], 2)
def test_SKDecimal():
space = SKDecimal(1, 2, decimals=2)

80
tests/plugins/test_pairlist.py
View File

@ -4,6 +4,7 @@ import time
from unittest.mock import MagicMock, PropertyMock
import pytest
import time_machine
from freqtrade.constants import AVAILABLE_PAIRLISTS
from freqtrade.exceptions import OperationalException
@ -815,32 +816,63 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick
def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history):
ohlcv_data = {
('ETH/BTC', '1d'): ohlcv_history,
('TKN/BTC', '1d'): ohlcv_history,
('LTC/BTC', '1d'): ohlcv_history,
}
mocker.patch.multiple('freqtrade.exchange.Exchange',
markets=PropertyMock(return_value=markets),
exchange_has=MagicMock(return_value=True),
get_tickers=tickers
)
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
)
with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
ohlcv_data = {
('ETH/BTC', '1d'): ohlcv_history,
('TKN/BTC', '1d'): ohlcv_history,
('LTC/BTC', '1d'): ohlcv_history,
}
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
markets=PropertyMock(return_value=markets),
exchange_has=MagicMock(return_value=True),
get_tickers=tickers,
refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data),
)
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter)
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 3
assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter)
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 3
assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0
previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 3
# Called once for XRP/BTC
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 3
# Call to XRP/BTC cached
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2
ohlcv_data = {
('ETH/BTC', '1d'): ohlcv_history,
('TKN/BTC', '1d'): ohlcv_history,
('LTC/BTC', '1d'): ohlcv_history,
('XRP/BTC', '1d'): ohlcv_history.iloc[[0]],
}
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 3
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
# Move to next day
t.move_to("2021-09-02 01:00:00 +00:00")
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 3
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
# Move another day with fresh mocks (now the pair is old enough)
t.move_to("2021-09-03 01:00:00 +00:00")
# Called once for XRP/BTC
ohlcv_data = {
('ETH/BTC', '1d'): ohlcv_history,
('TKN/BTC', '1d'): ohlcv_history,
('LTC/BTC', '1d'): ohlcv_history,
('XRP/BTC', '1d'): ohlcv_history,
}
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data)
freqtrade.pairlists.refresh_pairlist()
assert len(freqtrade.pairlists.whitelist) == 4
# Called once (only for XRP/BTC)
assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 1
def test_OffsetFilter_error(mocker, whitelist_conf) -> None:

32
tests/test_periodiccache.py Normal file
View File

@ -0,0 +1,32 @@
import time_machine
from freqtrade.configuration import PeriodicCache
def test_ttl_cache():
with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
cache = PeriodicCache(5, ttl=60)
cache1h = PeriodicCache(5, ttl=3600)
assert cache.timer() == 1630472400.0
cache['a'] = 1235
cache1h['a'] = 555123
assert 'a' in cache
assert 'a' in cache1h
t.move_to("2021-09-01 05:00:59 +00:00")
assert 'a' in cache
assert 'a' in cache1h
# Cache expired
t.move_to("2021-09-01 05:01:00 +00:00")
assert 'a' not in cache
assert 'a' in cache1h
t.move_to("2021-09-01 05:59:59 +00:00")
assert 'a' in cache1h
t.move_to("2021-09-01 06:00:00 +00:00")
assert 'a' not in cache1h