adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #7289 from freqtrade/feat/freqai-rl-dev

Browse Source 
Add reinforcement learning module to FreqAI
This commit is contained in:
Matthias
2022-11-27 17:15:21 +01:00
committed by GitHub
parent 2219d2f491 732757e087
commit f4025ee5de
27 changed files with 1874 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
tests/freqai/conftest.py
View File

@@ -27,10 +27,9 @@ def freqai_conf(default_conf, tmpdir):
"timerange": "20180110-20180115",
"freqai": {
"enabled": True,
"startup_candles": 10000,
"purge_old_models": True,
"train_period_days": 2,
"backtest_period_days": 2,
"backtest_period_days": 10,
"live_retrain_hours": 0,
"expiration_hours": 1,
"identifier": "uniqe-id100",
@@ -58,6 +57,30 @@ def freqai_conf(default_conf, tmpdir):
return freqaiconf
def make_rl_config(conf):
conf.update({"strategy": "freqai_rl_test_strat"})
conf["freqai"].update({"model_training_parameters": {
"learning_rate": 0.00025,
"gamma": 0.9,
"verbose": 1
}})
conf["freqai"]["rl_config"] = {
"train_cycles": 1,
"thread_count": 2,
"max_trade_duration_candles": 300,
"model_type": "PPO",
"policy_type": "MlpPolicy",
"max_training_drawdown_pct": 0.5,
"net_arch": [32, 32],
"model_reward_parameters": {
"rr": 1,
"profit_aim": 0.02,
"win_reward_factor": 2
}}
return conf
def get_patched_data_kitchen(mocker, freqaiconf):
dk = FreqaiDataKitchen(freqaiconf)
return dk

100
tests/freqai/test_freqai_interface.py
View File

@@ -13,8 +13,8 @@ from freqtrade.freqai.utils import download_all_data_for_training, get_required_
from freqtrade.optimize.backtesting import Backtesting
from freqtrade.persistence import Trade
from freqtrade.plugins.pairlistmanager import PairListManager
from tests.conftest import get_patched_exchange, log_has_re
from tests.freqai.conftest import get_patched_freqai_strategy
from tests.conftest import create_mock_trades, get_patched_exchange, log_has_re
from tests.freqai.conftest import get_patched_freqai_strategy, make_rl_config
def is_arm() -> bool:
@@ -32,11 +32,17 @@ def is_mac() -> bool:
('XGBoostRegressor', False, True, False),
('XGBoostRFRegressor', False, False, False),
('CatboostRegressor', False, False, False),
('ReinforcementLearner', False, True, False),
('ReinforcementLearner_multiproc', False, False, False),
('ReinforcementLearner_test_4ac', False, False, False)
])
def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, dbscan, float32):
if is_arm() and model == 'CatboostRegressor':
pytest.skip("CatBoost is not supported on ARM")
if is_mac() and 'Reinforcement' in model:
pytest.skip("Reinforcement learning module not available on intel based Mac OS")
model_save_ext = 'joblib'
freqai_conf.update({"freqaimodel": model})
freqai_conf.update({"timerange": "20180110-20180130"})
@@ -45,6 +51,26 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca,
freqai_conf['freqai']['feature_parameters'].update({"use_DBSCAN_to_remove_outliers": dbscan})
freqai_conf.update({"reduce_df_footprint": float32})
if 'ReinforcementLearner' in model:
model_save_ext = 'zip'
freqai_conf = make_rl_config(freqai_conf)
# test the RL guardrails
freqai_conf['freqai']['feature_parameters'].update({"use_SVM_to_remove_outliers": True})
freqai_conf['freqai']['data_split_parameters'].update({'shuffle': True})
if 'test_4ac' in model:
freqai_conf["freqaimodel_path"] = str(Path(__file__).parents[1] / "freqai" / "test_models")
if 'ReinforcementLearner' in model:
model_save_ext = 'zip'
freqai_conf = make_rl_config(freqai_conf)
# test the RL guardrails
freqai_conf['freqai']['feature_parameters'].update({"use_SVM_to_remove_outliers": True})
freqai_conf['freqai']['data_split_parameters'].update({'shuffle': True})
if 'test_4ac' in model:
freqai_conf["freqaimodel_path"] = str(Path(__file__).parents[1] / "freqai" / "test_models")
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
@@ -52,6 +78,7 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca,
freqai = strategy.freqai
freqai.live = True
freqai.dk = FreqaiDataKitchen(freqai_conf)
freqai.dk.set_paths('ADA/BTC', 10000)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
@@ -165,25 +192,35 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model):
@pytest.mark.parametrize(
"model, num_files, strat",
[
("LightGBMRegressor", 6, "freqai_test_strat"),
("XGBoostRegressor", 6, "freqai_test_strat"),
("CatboostRegressor", 6, "freqai_test_strat"),
("XGBoostClassifier", 6, "freqai_test_classifier"),
("LightGBMClassifier", 6, "freqai_test_classifier"),
("CatboostClassifier", 6, "freqai_test_classifier")
("LightGBMRegressor", 2, "freqai_test_strat"),
("XGBoostRegressor", 2, "freqai_test_strat"),
("CatboostRegressor", 2, "freqai_test_strat"),
("ReinforcementLearner", 3, "freqai_rl_test_strat"),
("XGBoostClassifier", 2, "freqai_test_classifier"),
("LightGBMClassifier", 2, "freqai_test_classifier"),
("CatboostClassifier", 2, "freqai_test_classifier")
],
)
def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog):
freqai_conf.get("freqai", {}).update({"save_backtest_models": True})
freqai_conf['runmode'] = RunMode.BACKTEST
Trade.use_db = False
if is_arm() and "Catboost" in model:
pytest.skip("CatBoost is not supported on ARM")
if is_mac() and 'Reinforcement' in model:
pytest.skip("Reinforcement learning module not available on intel based Mac OS")
Trade.use_db = False
freqai_conf.update({"freqaimodel": model})
freqai_conf.update({"timerange": "20180120-20180130"})
freqai_conf.update({"strategy": strat})
if 'ReinforcementLearner' in model:
freqai_conf = make_rl_config(freqai_conf)
if 'test_4ac' in model:
freqai_conf["freqaimodel_path"] = str(Path(__file__).parents[1] / "freqai" / "test_models")
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
@@ -207,6 +244,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog)
model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()]
assert len(model_folders) == num_files
Trade.use_db = True
assert log_has_re(
"Removed features ",
caplog,
@@ -267,7 +305,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog):
freqai.start_backtesting(df, metadata, freqai.dk)
model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()]
assert len(model_folders) == 6
assert len(model_folders) == 2
# without deleting the existing folder structure, re-run
@@ -295,7 +333,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog):
path = (freqai.dd.full_path / freqai.dk.backtest_predictions_folder)
prediction_files = [x for x in path.iterdir() if x.is_file()]
assert len(prediction_files) == 5
assert len(prediction_files) == 1
shutil.rmtree(Path(freqai.dk.full_path))
@@ -473,3 +511,43 @@ def test_download_all_data_for_training(mocker, freqai_conf, caplog, tmpdir):
"Downloading",
caplog,
)
@pytest.mark.usefixtures("init_persistence")
@pytest.mark.parametrize('dp_exists', [(False), (True)])
def test_get_state_info(mocker, freqai_conf, dp_exists, caplog, tickers):
if is_mac():
pytest.skip("Reinforcement learning module not available on intel based Mac OS")
freqai_conf.update({"freqaimodel": "ReinforcementLearner"})
freqai_conf.update({"timerange": "20180110-20180130"})
freqai_conf.update({"strategy": "freqai_rl_test_strat"})
freqai_conf = make_rl_config(freqai_conf)
freqai_conf['entry_pricing']['price_side'] = 'same'
freqai_conf['exit_pricing']['price_side'] = 'same'
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
ticker_mock = MagicMock(return_value=tickers()['ETH/BTC'])
mocker.patch("freqtrade.exchange.Exchange.fetch_ticker", ticker_mock)
strategy.dp = DataProvider(freqai_conf, exchange)
if not dp_exists:
strategy.dp._exchange = None
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.data_provider = strategy.dp
freqai.live = True
Trade.use_db = True
create_mock_trades(MagicMock(return_value=0.0025), False, True)
freqai.get_state_info("ADA/BTC")
freqai.get_state_info("ETH/BTC")
if not dp_exists:
assert log_has_re(
"No exchange available",
caplog,
)

66
tests/freqai/test_models/ReinforcementLearner_test_4ac.py Normal file
View File

@@ -0,0 +1,66 @@
import logging
import numpy as np
from freqtrade.freqai.prediction_models.ReinforcementLearner import ReinforcementLearner
from freqtrade.freqai.RL.Base4ActionRLEnv import Actions, Base4ActionRLEnv, Positions
logger = logging.getLogger(__name__)
class ReinforcementLearner_test_4ac(ReinforcementLearner):
"""
User created Reinforcement Learning Model prediction model.
"""
class MyRLEnv(Base4ActionRLEnv):
"""
User can override any function in BaseRLEnv and gym.Env. Here the user
sets a custom reward based on profit and trade duration.
"""
def calculate_reward(self, action: int) -> float:
# first, penalize if the action is not valid
if not self._is_valid(action):
return -2
pnl = self.get_unrealized_profit()
rew = np.sign(pnl) * (pnl + 1)
factor = 100.
# reward agent for entering trades
if (action in (Actions.Long_enter.value, Actions.Short_enter.value)
and self._position == Positions.Neutral):
return 25
# discourage agent from not entering trades
if action == Actions.Neutral.value and self._position == Positions.Neutral:
return -1
max_trade_duration = self.rl_config.get('max_trade_duration_candles', 300)
trade_duration = self._current_tick - self._last_trade_tick # type: ignore
if trade_duration <= max_trade_duration:
factor *= 1.5
elif trade_duration > max_trade_duration:
factor *= 0.5
# discourage sitting in position
if (self._position in (Positions.Short, Positions.Long) and
action == Actions.Neutral.value):
return -1 * trade_duration / max_trade_duration
# close long
if action == Actions.Exit.value and self._position == Positions.Long:
if pnl > self.profit_aim * self.rr:
factor *= self.rl_config['model_reward_parameters'].get('win_reward_factor', 2)
return float(rew * factor)
# close short
if action == Actions.Exit.value and self._position == Positions.Short:
if pnl > self.profit_aim * self.rr:
factor *= self.rl_config['model_reward_parameters'].get('win_reward_factor', 2)
return float(rew * factor)
return 0.

1
tests/rpc/test_rpc_apiserver.py
View File

@@ -1461,6 +1461,7 @@ def test_api_strategies(botclient, tmpdir):
'StrategyTestV3',
'StrategyTestV3CustomEntryPrice',
'StrategyTestV3Futures',
'freqai_rl_test_strat',
'freqai_test_classifier',
'freqai_test_multimodel_classifier_strat',
'freqai_test_multimodel_strat',

105
tests/strategy/strats/freqai_rl_test_strat.py Normal file
View File

@@ -0,0 +1,105 @@
import logging
from functools import reduce
import pandas as pd
import talib.abstract as ta
from pandas import DataFrame
from freqtrade.strategy import IStrategy, merge_informative_pair
logger = logging.getLogger(__name__)
class freqai_rl_test_strat(IStrategy):
"""
Test strategy - used for testing freqAI functionalities.
DO not use in production.
"""
minimal_roi = {"0": 0.1, "240": -1}
process_only_new_candles = True
stoploss = -0.05
use_exit_signal = True
startup_candle_count: int = 30
can_short = False
def populate_any_indicators(
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
if informative is None:
informative = self.dp.get_pair_dataframe(pair, tf)
# first loop is automatically duplicating indicators for time periods
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t)
informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
# The following columns are necessary for RL models.
informative[f"%-{pair}raw_close"] = informative["close"]
informative[f"%-{pair}raw_open"] = informative["open"]
informative[f"%-{pair}raw_high"] = informative["high"]
informative[f"%-{pair}raw_low"] = informative["low"]
indicators = [col for col in informative if col.startswith("%")]
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True)
skip_columns = [
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
# Add generalized indicators here (because in live, it will call this
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators:
# For RL, there are no direct targets to set. This is filler (neutral)
# until the agent sends an action.
df["&-action"] = 0
return df
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe = self.freqai.start(dataframe, metadata, self)
return dataframe
def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame:
enter_long_conditions = [df["do_predict"] == 1, df["&-action"] == 1]
if enter_long_conditions:
df.loc[
reduce(lambda x, y: x & y, enter_long_conditions), ["enter_long", "enter_tag"]
] = (1, "long")
enter_short_conditions = [df["do_predict"] == 1, df["&-action"] == 3]
if enter_short_conditions:
df.loc[
reduce(lambda x, y: x & y, enter_short_conditions), ["enter_short", "enter_tag"]
] = (1, "short")
return df
def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame:
exit_long_conditions = [df["do_predict"] == 1, df["&-action"] == 2]
if exit_long_conditions:
df.loc[reduce(lambda x, y: x & y, exit_long_conditions), "exit_long"] = 1
exit_short_conditions = [df["do_predict"] == 1, df["&-action"] == 4]
if exit_short_conditions:
df.loc[reduce(lambda x, y: x & y, exit_short_conditions), "exit_short"] = 1
return df

6
tests/strategy/test_strategy_loading.py
View File

@@ -34,7 +34,7 @@ def test_search_all_strategies_no_failed():
directory = Path(__file__).parent / "strats"
strategies = StrategyResolver._search_all_objects(directory, enum_failed=False)
assert isinstance(strategies, list)
assert len(strategies) == 11
assert len(strategies) == 12
assert isinstance(strategies[0], dict)
@@ -42,10 +42,10 @@ def test_search_all_strategies_with_failed():
directory = Path(__file__).parent / "strats"
strategies = StrategyResolver._search_all_objects(directory, enum_failed=True)
assert isinstance(strategies, list)
assert len(strategies) == 12
assert len(strategies) == 13
# with enum_failed=True search_all_objects() shall find 2 good strategies
# and 1 which fails to load
assert len([x for x in strategies if x['class'] is not None]) == 11
assert len([x for x in strategies if x['class'] is not None]) == 12
assert len([x for x in strategies if x['class'] is None]) == 1