adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change config parameter names to improve clarity and consistency throughout the code (!!breaking change, please check discord support channel for migration instructions or review templates/FreqaiExampleStrategy.py config_examples/config_freqai_futures.example.json file changes!!)

Browse Source
This commit is contained in:
Robert Caulk
2022-07-10 12:34:09 +02:00
parent 819cc9c0e4
commit 607455919e
10 changed files with 269 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
freqtrade/configuration/config_validation.py
View File

@@ -174,9 +174,10 @@ def _validate_freqai(conf: Dict[str, Any]) -> None:
for param in constants.SCHEMA_FREQAI_REQUIRED:
if param not in conf.get('freqai', {}):
raise OperationalException(
f'{param} not found in Freqai config'
)
if param not in conf.get('freqai', {}).get('feature_parameters', {}):
raise OperationalException(
f'{param} not found in Freqai config'
)
def _validate_whitelist(conf: Dict[str, Any]) -> None:

20
freqtrade/constants.py
View File

@@ -477,16 +477,16 @@ CONF_SCHEMA = {
"freqai": {
"type": "object",
"properties": {
"timeframes": {"type": "list"},
"train_period": {"type": "integer", "default": 0},
"backtest_period": {"type": "float", "default": 7},
"train_period_days": {"type": "integer", "default": 0},
"backtest_period_days": {"type": "float", "default": 7},
"identifier": {"type": "str", "default": "example"},
"corr_pairlist": {"type": "list"},
"feature_parameters": {
"type": "object",
"properties": {
"period": {"type": "integer"},
"shift": {"type": "integer", "default": 0},
"include_corr_pairlist": {"type": "list"},
"include_timeframes": {"type": "list"},
"label_period_candles": {"type": "integer"},
"include_shifted_candles": {"type": "integer", "default": 0},
"DI_threshold": {"type": "float", "default": 0},
"weight_factor": {"type": "number", "default": 0},
"principal_component_analysis": {"type": "boolean", "default": False},
@@ -555,11 +555,11 @@ SCHEMA_MINIMAL_REQUIRED = [
]
SCHEMA_FREQAI_REQUIRED = [
'timeframes',
'train_period',
'backtest_period',
'include_timeframes',
'train_period_days',
'backtest_period_days',
'identifier',
'corr_pairlist',
'include_corr_pairlist',
'feature_parameters',
'data_split_parameters',
'model_training_parameters'

64
freqtrade/freqai/data_kitchen.py
View File

@@ -26,6 +26,7 @@ from freqtrade.strategy.interface import IStrategy
SECONDS_IN_DAY = 86400
SECONDS_IN_HOUR = 3600
logger = logging.getLogger(__name__)
@@ -59,13 +60,13 @@ class FreqaiDataKitchen:
self.set_all_pairs()
if not self.live:
self.full_timerange = self.create_fulltimerange(
self.config["timerange"], self.freqai_config.get("train_period")
self.config["timerange"], self.freqai_config.get("train_period_days")
)
(self.training_timeranges, self.backtesting_timeranges) = self.split_timerange(
self.full_timerange,
config["freqai"]["train_period"],
config["freqai"]["backtest_period"],
config["freqai"]["train_period_days"],
config["freqai"]["backtest_period_days"],
)
# self.strat_dataframe: DataFrame = strat_dataframe
self.dd = data_drawer
@@ -234,17 +235,18 @@ class FreqaiDataKitchen:
:filtered_dataframe: cleaned dataframe ready to be split.
:labels: cleaned labels ready to be split.
"""
feat_dict = self.freqai_config.get("feature_parameters", {})
weights: npt.ArrayLike
if self.freqai_config["feature_parameters"].get("weight_factor", 0) > 0:
if feat_dict.get("weight_factor", 0) > 0:
weights = self.set_weights_higher_recent(len(filtered_dataframe))
else:
weights = np.ones(len(filtered_dataframe))
if self.freqai_config["feature_parameters"].get("stratify", 0) > 0:
if feat_dict.get("stratify_training_data", 0) > 0:
stratification = np.zeros(len(filtered_dataframe))
for i in range(1, len(stratification)):
if i % self.freqai_config.get("feature_parameters", {}).get("stratify", 0) == 0:
if i % feat_dict.get("stratify_training_data", 0) == 0:
stratification[i] = 1
else:
stratification = None
@@ -439,7 +441,7 @@ class FreqaiDataKitchen:
bt_split: the backtesting length (dats). Specified in user configuration file
"""
train_period = train_split * SECONDS_IN_DAY
train_period_days = train_split * SECONDS_IN_DAY
bt_period = bt_split * SECONDS_IN_DAY
full_timerange = TimeRange.parse_timerange(tr)
@@ -460,7 +462,7 @@ class FreqaiDataKitchen:
while True:
if not first:
timerange_train.startts = timerange_train.startts + bt_period
timerange_train.stopts = timerange_train.startts + train_period
timerange_train.stopts = timerange_train.startts + train_period_days
first = False
start = datetime.datetime.utcfromtimestamp(timerange_train.startts)
@@ -763,7 +765,7 @@ class FreqaiDataKitchen:
return
def create_fulltimerange(self, backtest_tr: str, backtest_period: int) -> str:
def create_fulltimerange(self, backtest_tr: str, backtest_period_days: int) -> str:
backtest_timerange = TimeRange.parse_timerange(backtest_tr)
if backtest_timerange.stopts == 0:
@@ -771,7 +773,8 @@ class FreqaiDataKitchen:
datetime.datetime.now(tz=datetime.timezone.utc).timestamp()
)
backtest_timerange.startts = backtest_timerange.startts - backtest_period * SECONDS_IN_DAY
backtest_timerange.startts = (backtest_timerange.startts
- backtest_period_days * SECONDS_IN_DAY)
start = datetime.datetime.utcfromtimestamp(backtest_timerange.startts)
stop = datetime.datetime.utcfromtimestamp(backtest_timerange.stopts)
full_timerange = start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d")
@@ -817,7 +820,8 @@ class FreqaiDataKitchen:
data_load_timerange = TimeRange()
# find the max indicator length required
max_timeframe_chars = self.freqai_config.get("timeframes")[-1]
max_timeframe_chars = self.freqai_config.get(
"feature_parameters", {}).get("include_timeframes")[-1]
max_period = self.freqai_config.get("feature_parameters", {}).get(
"indicator_max_period", 50
)
@@ -840,11 +844,11 @@ class FreqaiDataKitchen:
# logger.info(f'Extending data download by {additional_seconds/SECONDS_IN_DAY:.2f} days')
if trained_timestamp != 0:
elapsed_time = (time - trained_timestamp) / SECONDS_IN_DAY
retrain = elapsed_time > self.freqai_config.get("backtest_period")
elapsed_time = (time - trained_timestamp) / SECONDS_IN_HOUR
retrain = elapsed_time > self.freqai_config.get("live_retrain_hours", 0)
if retrain:
trained_timerange.startts = int(
time - self.freqai_config.get("train_period", 0) * SECONDS_IN_DAY
time - self.freqai_config.get("train_period_days", 0) * SECONDS_IN_DAY
)
trained_timerange.stopts = int(time)
# we want to load/populate indicators on more data than we plan to train on so
@@ -852,19 +856,19 @@ class FreqaiDataKitchen:
# unless they have data further back in time before the start of the train period
data_load_timerange.startts = int(
time
- self.freqai_config.get("train_period", 0) * SECONDS_IN_DAY
- self.freqai_config.get("train_period_days", 0) * SECONDS_IN_DAY
- additional_seconds
)
data_load_timerange.stopts = int(time)
else: # user passed no live_trained_timerange in config
trained_timerange.startts = int(
time - self.freqai_config.get("train_period") * SECONDS_IN_DAY
time - self.freqai_config.get("train_period_days") * SECONDS_IN_DAY
)
trained_timerange.stopts = int(time)
data_load_timerange.startts = int(
time
- self.freqai_config.get("train_period", 0) * SECONDS_IN_DAY
- self.freqai_config.get("train_period_days", 0) * SECONDS_IN_DAY
- additional_seconds
)
data_load_timerange.stopts = int(time)
@@ -930,7 +934,7 @@ class FreqaiDataKitchen:
refresh_backtest_ohlcv_data(
exchange,
pairs=self.all_pairs,
timeframes=self.freqai_config.get("timeframes"),
timeframes=self.freqai_config.get("feature_parameters", {}).get("include_timeframes"),
datadir=self.config["datadir"],
timerange=timerange,
new_pairs_days=new_pairs_days,
@@ -948,12 +952,12 @@ class FreqaiDataKitchen:
:params:
dataframe: DataFrame = strategy provided dataframe
"""
feat_params = self.freqai_config.get("feature_parameters", {})
with self.dd.history_lock:
history_data = self.dd.historic_data
for pair in self.all_pairs:
for tf in self.freqai_config.get("timeframes"):
for tf in feat_params.get("include_timeframes"):
# check if newest candle is already appended
df_dp = strategy.dp.get_pair_dataframe(pair, tf)
@@ -992,7 +996,8 @@ class FreqaiDataKitchen:
def set_all_pairs(self) -> None:
self.all_pairs = copy.deepcopy(self.freqai_config.get("corr_pairlist", []))
self.all_pairs = copy.deepcopy(self.freqai_config.get(
'feature_parameters', {}).get('include_corr_pairlist', []))
for pair in self.config.get("exchange", "").get("pair_whitelist"):
if pair not in self.all_pairs:
self.all_pairs.append(pair)
@@ -1003,14 +1008,14 @@ class FreqaiDataKitchen:
Only called once upon startup of bot.
:params:
timerange: TimeRange = full timerange required to populate all indicators
for training according to user defined train_period
for training according to user defined train_period_days
"""
history_data = self.dd.historic_data
for pair in self.all_pairs:
if pair not in history_data:
history_data[pair] = {}
for tf in self.freqai_config.get("timeframes"):
for tf in self.freqai_config.get("feature_parameters", {}).get("include_timeframes"):
history_data[pair][tf] = load_pair_history(
datadir=self.config["datadir"],
timeframe=tf,
@@ -1028,7 +1033,7 @@ class FreqaiDataKitchen:
to the present pair.
:params:
timerange: TimeRange = full timerange required to populate all indicators
for training according to user defined train_period
for training according to user defined train_period_days
metadata: dict = strategy furnished pair metadata
"""
@@ -1036,9 +1041,10 @@ class FreqaiDataKitchen:
corr_dataframes: Dict[Any, Any] = {}
base_dataframes: Dict[Any, Any] = {}
historic_data = self.dd.historic_data
pairs = self.freqai_config.get("corr_pairlist", [])
pairs = self.freqai_config.get('feature_parameters', {}).get(
'include_corr_pairlist', [])
for tf in self.freqai_config.get("timeframes"):
for tf in self.freqai_config.get("feature_parameters", {}).get("include_timeframes"):
base_dataframes[tf] = self.slice_dataframe(timerange, historic_data[pair][tf])
if pairs:
for p in pairs:
@@ -1057,7 +1063,7 @@ class FreqaiDataKitchen:
# DataFrame]:
# corr_dataframes: Dict[Any, Any] = {}
# base_dataframes: Dict[Any, Any] = {}
# pairs = self.freqai_config.get('corr_pairlist', []) # + [metadata['pair']]
# pairs = self.freqai_config.get('include_corr_pairlist', []) # + [metadata['pair']]
# # timerange = TimeRange.parse_timerange(new_timerange)
# for tf in self.freqai_config.get('timeframes'):
@@ -1101,9 +1107,9 @@ class FreqaiDataKitchen:
dataframe: DataFrame = dataframe containing populated indicators
"""
dataframe = base_dataframes[self.config["timeframe"]].copy()
pairs = self.freqai_config.get("corr_pairlist", [])
pairs = self.freqai_config.get('feature_parameters', {}).get('include_corr_pairlist', [])
sgi = True
for tf in self.freqai_config.get("timeframes"):
for tf in self.freqai_config.get("feature_parameters", {}).get("include_timeframes"):
dataframe = strategy.populate_any_indicators(
pair,
pair,

12
freqtrade/freqai/freqai_interface.py
View File

@@ -95,7 +95,7 @@ class IFreqaiModel(ABC):
dk = self.start_live(dataframe, metadata, strategy, self.dk)
# For backtesting, each pair enters and then gets trained for each window along the
# sliding window defined by "train_period" (training window) and "backtest_period"
# sliding window defined by "train_period_days" (training window) and "live_retrain_hours"
# (backtest window, i.e. window immediately following the training window).
# FreqAI slides the window and sequentially builds the backtesting results before returning
# the concatenated results for the full backtesting period back to the strategy.
@@ -143,11 +143,11 @@ class IFreqaiModel(ABC):
) -> FreqaiDataKitchen:
"""
The main broad execution for backtesting. For backtesting, each pair enters and then gets
trained for each window along the sliding window defined by "train_period" (training window)
and "backtest_period" (backtest window, i.e. window immediately following the
training window). FreqAI slides the window and sequentially builds the backtesting results
before returning the concatenated results for the full backtesting period back to the
strategy.
trained for each window along the sliding window defined by "train_period_days"
(training window) and "backtest_period_days" (backtest window, i.e. window immediately
following the training window). FreqAI slides the window and sequentially builds
the backtesting results before returning the concatenated results for the full
backtesting period back to the strategy.
:params:
dataframe: DataFrame = strategy passed dataframe
metadata: Dict = pair metadata

21
freqtrade/freqai/prediction_models/CatboostPredictionModel.py
View File

@@ -27,29 +27,11 @@ class CatboostPredictionModel(IFreqaiModel):
return dataframe
def make_labels(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> DataFrame:
"""
User defines the labels here (target values).
:params:
:dataframe: the full dataframe for the present training period
"""
dataframe["s"] = (
dataframe["close"]
.shift(-self.feature_parameters["period"])
.rolling(self.feature_parameters["period"])
.mean()
/ dataframe["close"]
- 1
)
return dataframe["s"]
def train(
self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen
) -> Tuple[DataFrame, DataFrame]:
"""
Filter the training data and train a model to it. Train makes heavy use of the datahkitchen
Filter the training data and train a model to it. Train makes heavy use of the datakitchen
for storing, saving, loading, and analyzing the data.
:params:
:unfiltered_dataframe: Full dataframe for the current training period
@@ -60,7 +42,6 @@ class CatboostPredictionModel(IFreqaiModel):
logger.info("--------------------Starting training " f"{pair} --------------------")
# unfiltered_labels = self.make_labels(unfiltered_dataframe, dk)
# filter the features requested by user in the configuration file and elegantly handle NaNs
features_filtered, labels_filtered = dk.filter_features(
unfiltered_dataframe,

3
freqtrade/plugins/pairlist/pairlist_helpers.py
View File

@@ -44,7 +44,8 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
def dynamic_expand_pairlist(config: dict, markets: list) -> List[str]:
if config.get('freqai', {}):
full_pairs = config['pairs'] + [pair for pair in config['freqai']['corr_pairlist']
corr_pairlist = config['freqai']['feature_parameters']['include_corr_pairlist']
full_pairs = config['pairs'] + [pair for pair in corr_pairlist
if pair not in config['pairs']]
expanded_pairs = expand_pairlist(full_pairs, markets)
else:

18
freqtrade/templates/FreqaiExampleStrategy.py
View File

@@ -56,9 +56,9 @@ class FreqaiExampleStrategy(IStrategy):
def informative_pairs(self):
whitelist_pairs = self.dp.current_whitelist()
corr_pairs = self.config["freqai"]["corr_pairlist"]
corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"]
informative_pairs = []
for tf in self.config["freqai"]["timeframes"]:
for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]:
for pair in whitelist_pairs:
informative_pairs.append((pair, tf))
for pair in corr_pairs:
@@ -93,7 +93,7 @@ class FreqaiExampleStrategy(IStrategy):
informative = self.dp.get_pair_dataframe(pair, tf)
# first loop is automatically duplicating indicators for time periods
for t in self.freqai_info["feature_parameters"]["indicator_periods"]:
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t)
informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
@@ -123,8 +123,6 @@ class FreqaiExampleStrategy(IStrategy):
)
informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t)
macd = ta.MACD(informative, timeperiod=t)
informative[f"%-{coin}macd-period_{t}"] = macd["macd"]
informative[f"%-{coin}relative_volume-period_{t}"] = (
informative["volume"] / informative["volume"].rolling(t).mean()
@@ -136,7 +134,7 @@ class FreqaiExampleStrategy(IStrategy):
indicators = [col for col in informative if col.startswith("%")]
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["shift"] + 1):
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
@@ -161,8 +159,8 @@ class FreqaiExampleStrategy(IStrategy):
# needs to be used such as templates/CatboostPredictionMultiModel.py
df["&-s_close"] = (
df["close"]
.shift(-self.freqai_info["feature_parameters"]["period"])
.rolling(self.freqai_info["feature_parameters"]["period"])
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.mean()
/ df["close"]
- 1
@@ -179,7 +177,7 @@ class FreqaiExampleStrategy(IStrategy):
# indicated by the user in the configuration file.
# All indicators must be populated by populate_any_indicators() for live functionality
# to work correctly.
for tf in self.freqai_info["timeframes"]:
for tf in self.freqai_info["feature_parameters"]["include_timeframes"]:
dataframe = self.populate_any_indicators(
metadata,
self.pair,
@@ -189,7 +187,7 @@ class FreqaiExampleStrategy(IStrategy):
set_generalized_indicators=sgi,
)
sgi = False
for pair in self.freqai_info["corr_pairlist"]:
for pair in self.freqai_info["feature_parameters"]["include_corr_pairlist"]:
if metadata["pair"] in pair:
continue # do not include whitelisted pair twice if it is in corr_pairlist
dataframe = self.populate_any_indicators(