adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:lolongcovas/freqtrade into feat/freqai

Browse Source
This commit is contained in:
longyu
2022-08-17 14:32:36 +02:00
parent 8f9e6efc5e 96d2f61812
commit ffe7792251
39 changed files with 1089 additions and 936 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
freqtrade/freqai/data_drawer.py
View File

@@ -12,7 +12,7 @@ import pandas as pd
import rapidjson
from joblib import dump, load
from joblib.externals import cloudpickle
from numpy.typing import ArrayLike, NDArray
from numpy.typing import NDArray
from pandas import DataFrame
from freqtrade.configuration import TimeRange
@@ -38,8 +38,7 @@ class FreqaiDataDrawer:
"""
Class aimed at holding all pair models/info in memory for better inferencing/retrainig/saving
/loading to/from disk.
This object remains persistent throughout live/dry, unlike FreqaiDataKitchen, which is
reinstantiated for each coin.
This object remains persistent throughout live/dry.
Record of contribution:
FreqAI was developed by a group of individuals who all contributed specific skillsets to the
@@ -56,7 +55,7 @@ class FreqaiDataDrawer:
Beta testing and bug reporting:
@bloodhunter4rc, Salah Lamkadem @ikonx, @ken11o2, @longyu, @paranoidandy, @smidelis, @smarm
Juha Nykänen @suikula, Wagner Costa @wagnercosta
Juha Nykänen @suikula, Wagner Costa @wagnercosta, Johan Vlugt @Jooopieeert
"""
def __init__(self, full_path: Path, config: dict, follow_mode: bool = False):
@@ -85,6 +84,8 @@ class FreqaiDataDrawer:
self.load_historic_predictions_from_disk()
self.training_queue: Dict[str, int] = {}
self.history_lock = threading.Lock()
self.save_lock = threading.Lock()
self.pair_dict_lock = threading.Lock()
self.old_DBSCAN_eps: Dict[str, float] = {}
self.empty_pair_dict: pair_info = {
"model_filename": "", "trained_timestamp": 0,
@@ -145,9 +146,10 @@ class FreqaiDataDrawer:
"""
Save data drawer full of all pair model metadata in present model folder.
"""
with open(self.pair_dictionary_path, 'w') as fp:
rapidjson.dump(self.pair_dict, fp, default=self.np_encoder,
number_mode=rapidjson.NM_NATIVE)
with self.save_lock:
with open(self.pair_dictionary_path, 'w') as fp:
rapidjson.dump(self.pair_dict, fp, default=self.np_encoder,
number_mode=rapidjson.NM_NATIVE)
def save_follower_dict_to_disk(self):
"""
@@ -227,90 +229,50 @@ class FreqaiDataDrawer:
def pair_to_end_of_training_queue(self, pair: str) -> None:
# march all pairs up in the queue
for p in self.pair_dict:
self.pair_dict[p]["priority"] -= 1
# send pair to end of queue
self.pair_dict[pair]["priority"] = len(self.pair_dict)
with self.pair_dict_lock:
for p in self.pair_dict:
self.pair_dict[p]["priority"] -= 1
# send pair to end of queue
self.pair_dict[pair]["priority"] = len(self.pair_dict)
def set_initial_return_values(self, pair: str, dk: FreqaiDataKitchen,
pred_df: DataFrame, do_preds: ArrayLike) -> None:
def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None:
"""
Set the initial return values to a persistent dataframe. This avoids needing to repredict on
historical candles, and also stores historical predictions despite retrainings (so stored
predictions are true predictions, not just inferencing on trained data)
Set the initial return values to the historical predictions dataframe. This avoids needing
to repredict on historical candles, and also stores historical predictions despite
retrainings (so stored predictions are true predictions, not just inferencing on trained
data)
"""
# dynamic df returned to strategy and plotted in frequi
mrv_df = self.model_return_values[pair] = pd.DataFrame()
# if user reused `identifier` in config and has historical predictions collected, load them
# so that frequi remains uninterrupted after a crash
hist_df = self.historic_predictions
if pair in hist_df:
len_diff = len(hist_df[pair].index) - len(pred_df.index)
if len_diff < 0:
df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]],
ignore_index=True, keys=hist_df[pair].keys())
else:
df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True)
df_concat = df_concat.fillna(0)
self.model_return_values[pair] = df_concat
logger.info(f'Setting initial FreqUI plots from historical data for {pair}.')
len_diff = len(hist_df[pair].index) - len(pred_df.index)
if len_diff < 0:
df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]],
ignore_index=True, keys=hist_df[pair].keys())
else:
for label in pred_df.columns:
mrv_df[label] = pred_df[label]
if mrv_df[label].dtype == object:
continue
mrv_df[f"{label}_mean"] = dk.data["labels_mean"][label]
mrv_df[f"{label}_std"] = dk.data["labels_std"][label]
if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0:
mrv_df["DI_values"] = dk.DI_values
mrv_df["do_predict"] = do_preds
if dk.data['extra_returns_per_train']:
rets = dk.data['extra_returns_per_train']
for return_str in rets:
mrv_df[return_str] = rets[return_str]
# for keras type models, the conv_window needs to be prepended so
# viewing is correct in frequi
if self.freqai_info.get('keras', False):
n_lost_points = self.freqai_info.get('conv_width', 2)
zeros_df = DataFrame(np.zeros((n_lost_points, len(mrv_df.columns))),
columns=mrv_df.columns)
self.model_return_values[pair] = pd.concat(
[zeros_df, mrv_df], axis=0, ignore_index=True)
df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True)
df_concat = df_concat.fillna(0)
self.model_return_values[pair] = df_concat
def append_model_predictions(self, pair: str, predictions: DataFrame,
do_preds: NDArray[np.int_],
dk: FreqaiDataKitchen, len_df: int) -> None:
"""
Append model predictions to historic predictions dataframe, then set the
strategy return dataframe to the tail of the historic predictions. The length of
the tail is equivalent to the length of the dataframe that entered FreqAI from
the strategy originally. Doing this allows FreqUI to always display the correct
historic predictions.
"""
# strat seems to feed us variable sized dataframes - and since we are trying to build our
# own return array in the same shape, we need to figure out how the size has changed
# and adapt our stored/returned info accordingly.
index = self.historic_predictions[pair].index[-1:]
columns = self.historic_predictions[pair].columns
length_difference = len(self.model_return_values[pair]) - len_df
i = 0
nan_df = pd.DataFrame(np.nan, index=index, columns=columns)
self.historic_predictions[pair] = pd.concat(
[self.historic_predictions[pair], nan_df], ignore_index=True, axis=0)
df = self.historic_predictions[pair]
if length_difference == 0:
i = 1
elif length_difference > 0:
i = length_difference + 1
df = self.model_return_values[pair] = self.model_return_values[pair].shift(-i)
if pair in self.historic_predictions:
hp_df = self.historic_predictions[pair]
# here are some pandas hula hoops to accommodate the possibility of a series
# or dataframe depending number of labels requested by user
nan_df = pd.DataFrame(np.nan, index=hp_df.index[-2:] + 2, columns=hp_df.columns)
hp_df = pd.concat([hp_df, nan_df], ignore_index=True, axis=0)
self.historic_predictions[pair] = hp_df[:-1]
# incase user adds additional "predictions" e.g. predict_proba output:
# model outputs and associated statistics
for label in predictions.columns:
df[label].iloc[-1] = predictions[label].iloc[-1]
if df[label].dtype == object:
@@ -318,26 +280,18 @@ class FreqaiDataDrawer:
df[f"{label}_mean"].iloc[-1] = dk.data["labels_mean"][label]
df[f"{label}_std"].iloc[-1] = dk.data["labels_std"][label]
# outlier indicators
df["do_predict"].iloc[-1] = do_preds[-1]
if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0:
df["DI_values"].iloc[-1] = dk.DI_values[-1]
# extra values the user added within custom prediction model
if dk.data['extra_returns_per_train']:
rets = dk.data['extra_returns_per_train']
for return_str in rets:
df[return_str].iloc[-1] = rets[return_str]
# append the new predictions to persistent storage
if pair in self.historic_predictions:
for key in df.keys():
self.historic_predictions[pair][key].iloc[-1] = df[key].iloc[-1]
if length_difference < 0:
prepend_df = pd.DataFrame(
np.zeros((abs(length_difference) - 1, len(df.columns))), columns=df.columns
)
df = pd.concat([prepend_df, df], axis=0)
self.model_return_values[pair] = df.tail(len_df).reset_index(drop=True)
def attach_return_values_to_return_dataframe(
self, pair: str, dataframe: DataFrame) -> DataFrame:
@@ -358,10 +312,7 @@ class FreqaiDataDrawer:
dk.find_features(dataframe)
if self.freqai_info.get('predict_proba', []):
full_labels = dk.label_list + self.freqai_info['predict_proba']
else:
full_labels = dk.label_list
full_labels = dk.label_list + dk.unique_class_list
for label in full_labels:
dataframe[label] = 0
@@ -575,7 +526,7 @@ class FreqaiDataDrawer:
history_data[pair][tf] = pd.concat(
[
history_data[pair][tf],
strategy.dp.get_pair_dataframe(pair, tf).iloc[index:],
df_dp.iloc[index:],
],
ignore_index=True,
axis=0,

28
freqtrade/freqai/data_kitchen.py
View File

@@ -34,6 +34,9 @@ class FreqaiDataKitchen:
Class designed to analyze data for a single pair. Employed by the IFreqaiModel class.
Functionalities include holding, saving, loading, and analyzing the data.
This object is not persistent, it is reinstantiated for each coin, each time the coin
model needs to be inferenced or trained.
Record of contribution:
FreqAI was developed by a group of individuals who all contributed specific skillsets to the
project.
@@ -49,7 +52,7 @@ class FreqaiDataKitchen:
Beta testing and bug reporting:
@bloodhunter4rc, Salah Lamkadem @ikonx, @ken11o2, @longyu, @paranoidandy, @smidelis, @smarm
Juha Nykänen @suikula, Wagner Costa @wagnercosta
Juha Nykänen @suikula, Wagner Costa @wagnercosta, Johan Vlugt @Jooopieeert
"""
def __init__(
@@ -70,6 +73,7 @@ class FreqaiDataKitchen:
self.model_filename: str = ""
self.live = live
self.pair = pair
self.svm_model: linear_model.SGDOneClassSVM = None
self.keras: bool = self.freqai_config.get("keras", False)
self.set_all_pairs()
@@ -90,6 +94,8 @@ class FreqaiDataKitchen:
self.data['extra_returns_per_train'] = self.freqai_config.get('extra_returns_per_train', {})
self.thread_count = self.freqai_config.get("data_kitchen_thread_count", -1)
self.train_dates: DataFrame = pd.DataFrame()
self.unique_classes: Dict[str, list] = {}
self.unique_class_list: list = []
def set_paths(
self,
@@ -337,7 +343,7 @@ class FreqaiDataKitchen:
"""
for label in df.columns:
if df[label].dtype == object:
if df[label].dtype == object or label in self.unique_class_list:
continue
df[label] = (
(df[label] + 1)
@@ -977,6 +983,8 @@ class FreqaiDataKitchen:
informative=corr_dataframes[i][tf]
)
self.get_unique_classes_from_labels(dataframe)
return dataframe
def fit_labels(self) -> None:
@@ -992,6 +1000,10 @@ class FreqaiDataKitchen:
f = spy.stats.norm.fit(self.data_dictionary["train_labels"][label])
self.data["labels_mean"][label], self.data["labels_std"][label] = f[0], f[1]
# incase targets are classifications
for label in self.unique_class_list:
self.data["labels_mean"][label], self.data["labels_std"][label] = 0, 0
return
def remove_features_from_df(self, dataframe: DataFrame) -> DataFrame:
@@ -1003,3 +1015,15 @@ class FreqaiDataKitchen:
col for col in dataframe.columns if not col.startswith("%") or col.startswith("%%")
]
return dataframe[to_keep]
def get_unique_classes_from_labels(self, dataframe: DataFrame) -> None:
self.find_features(dataframe)
for key in self.label_list:
if dataframe[key].dtype == object:
self.unique_classes[key] = dataframe[key].dropna().unique()
if self.unique_classes:
for label in self.unique_classes:
self.unique_class_list += list(self.unique_classes[label])

94
freqtrade/freqai/freqai_interface.py
View File

@@ -6,6 +6,7 @@ import threading
import time
from abc import ABC, abstractmethod
from pathlib import Path
from threading import Lock
from typing import Any, Dict, Tuple
import numpy as np
@@ -16,6 +17,7 @@ from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.enums import RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_seconds
from freqtrade.freqai.data_drawer import FreqaiDataDrawer
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
from freqtrade.strategy.interface import IStrategy
@@ -52,7 +54,7 @@ class IFreqaiModel(ABC):
Beta testing and bug reporting:
@bloodhunter4rc, Salah Lamkadem @ikonx, @ken11o2, @longyu, @paranoidandy, @smidelis, @smarm
Juha Nykänen @suikula, Wagner Costa @wagnercosta
Juha Nykänen @suikula, Wagner Costa @wagnercosta, Johan Vlugt @Jooopieeert
"""
def __init__(self, config: Dict[str, Any]) -> None:
@@ -70,7 +72,6 @@ class IFreqaiModel(ABC):
self.set_full_path()
self.follow_mode: bool = self.freqai_info.get("follow_mode", False)
self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode)
self.lock = threading.Lock()
self.identifier: str = self.freqai_info.get("identifier", "no_id_provided")
self.scanning = False
self.keras: bool = self.freqai_info.get("keras", False)
@@ -82,6 +83,10 @@ class IFreqaiModel(ABC):
self.total_pairs = len(self.config.get("exchange", {}).get("pair_whitelist"))
self.last_trade_database_summary: DataFrame = {}
self.current_trade_database_summary: DataFrame = {}
self.analysis_lock = Lock()
self.inference_time: float = 0
self.begin_time: float = 0
self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe'])
def assert_config(self, config: Dict[str, Any]) -> None:
@@ -104,6 +109,7 @@ class IFreqaiModel(ABC):
self.dd.set_pair_dict_info(metadata)
if self.live:
self.inference_timer('start')
self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
dk = self.start_live(dataframe, metadata, strategy, self.dk)
@@ -115,14 +121,16 @@ class IFreqaiModel(ABC):
elif not self.follow_mode:
self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
logger.info(f"Training {len(self.dk.training_timeranges)} timeranges")
dataframe = self.dk.use_strategy_to_populate_indicators(
strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
)
with self.analysis_lock:
dataframe = self.dk.use_strategy_to_populate_indicators(
strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
)
dk = self.start_backtesting(dataframe, metadata, self.dk)
dataframe = dk.remove_features_from_df(dk.return_dataframe)
del dk
if self.live:
self.inference_timer('stop')
return dataframe
@threaded
@@ -155,6 +163,8 @@ class IFreqaiModel(ABC):
new_trained_timerange, pair, strategy, dk, data_load_timerange
)
self.dd.save_historic_predictions_to_disk()
def start_backtesting(
self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen
) -> FreqaiDataKitchen:
@@ -290,9 +300,10 @@ class IFreqaiModel(ABC):
# load the model and associated data into the data kitchen
self.model = self.dd.load_data(metadata["pair"], dk)
dataframe = self.dk.use_strategy_to_populate_indicators(
strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
)
with self.analysis_lock:
dataframe = self.dk.use_strategy_to_populate_indicators(
strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
)
if not self.model:
logger.warning(
@@ -319,7 +330,10 @@ class IFreqaiModel(ABC):
# first predictions are made on entire historical candle set coming from strategy. This
# allows FreqUI to show full return values.
pred_df, do_preds = self.predict(dataframe, dk)
self.dd.set_initial_return_values(pair, dk, pred_df, do_preds)
if pair not in self.dd.historic_predictions:
self.set_initial_historic_predictions(pred_df, dk, pair)
self.dd.set_initial_return_values(pair, pred_df)
dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
return
elif self.dk.check_if_model_expired(trained_timestamp):
@@ -336,6 +350,8 @@ class IFreqaiModel(ABC):
# historical accuracy reasons.
pred_df, do_preds = self.predict(dataframe.iloc[-self.CONV_WIDTH:], dk, first=False)
if self.freqai_info.get('fit_live_predictions_candles', 0) and self.live:
self.fit_live_predictions(dk, pair)
self.dd.append_model_predictions(pair, pred_df, do_preds, dk, len(dataframe))
dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
@@ -480,9 +496,10 @@ class IFreqaiModel(ABC):
data_load_timerange, pair, dk
)
unfiltered_dataframe = dk.use_strategy_to_populate_indicators(
strategy, corr_dataframes, base_dataframes, pair
)
with self.analysis_lock:
unfiltered_dataframe = dk.use_strategy_to_populate_indicators(
strategy, corr_dataframes, base_dataframes, pair
)
unfiltered_dataframe = dk.slice_dataframe(new_trained_timerange, unfiltered_dataframe)
@@ -495,15 +512,14 @@ class IFreqaiModel(ABC):
dk.set_new_model_names(pair, new_trained_timerange)
self.dd.pair_dict[pair]["first"] = False
if self.dd.pair_dict[pair]["priority"] == 1 and self.scanning:
with self.lock:
self.dd.pair_to_end_of_training_queue(pair)
self.dd.pair_to_end_of_training_queue(pair)
self.dd.save_data(model, pair, dk)
if self.freqai_info.get("purge_old_models", False):
self.dd.purge_old_models()
def set_initial_historic_predictions(
self, df: DataFrame, model: Any, dk: FreqaiDataKitchen, pair: str
self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str
) -> None:
"""
This function is called only if the datadrawer failed to load an
@@ -528,14 +544,6 @@ class IFreqaiModel(ABC):
:param: dk: FreqaiDataKitchen = object containing methods for data analysis
:param: pair: str = current pair
"""
num_candles = self.freqai_info.get('fit_live_predictions_candles', 600)
if not num_candles:
num_candles = 600
df_tail = df.tail(num_candles)
trained_predictions = model.predict(df_tail)
pred_df = DataFrame(trained_predictions, columns=dk.label_list)
pred_df = dk.denormalize_labels_from_metadata(pred_df)
self.dd.historic_predictions[pair] = pred_df
hist_preds_df = self.dd.historic_predictions[pair]
@@ -554,15 +562,27 @@ class IFreqaiModel(ABC):
for return_str in dk.data['extra_returns_per_train']:
hist_preds_df[return_str] = 0
def fit_live_predictions(self, dk: FreqaiDataKitchen) -> None:
# # for keras type models, the conv_window needs to be prepended so
# # viewing is correct in frequi
if self.freqai_info.get('keras', False):
n_lost_points = self.freqai_info.get('conv_width', 2)
zeros_df = DataFrame(np.zeros((n_lost_points, len(hist_preds_df.columns))),
columns=hist_preds_df.columns)
self.dd.historic_predictions[pair] = pd.concat(
[zeros_df, hist_preds_df], axis=0, ignore_index=True)
def fit_live_predictions(self, dk: FreqaiDataKitchen, pair: str) -> None:
"""
Fit the labels with a gaussian distribution
"""
import scipy as spy
# add classes from classifier label types if used
full_labels = dk.label_list + dk.unique_class_list
num_candles = self.freqai_info.get("fit_live_predictions_candles", 100)
dk.data["labels_mean"], dk.data["labels_std"] = {}, {}
for label in dk.label_list:
for label in full_labels:
if self.dd.historic_predictions[dk.pair][label].dtype == object:
continue
f = spy.stats.norm.fit(self.dd.historic_predictions[dk.pair][label].tail(num_candles))
@@ -570,6 +590,28 @@ class IFreqaiModel(ABC):
return
def inference_timer(self, do='start'):
"""
Timer designed to track the cumulative time spent in FreqAI for one pass through
the whitelist. This will check if the time spent is more than 1/4 the time
of a single candle, and if so, it will warn the user of degraded performance
"""
if do == 'start':
self.pair_it += 1
self.begin_time = time.time()
elif do == 'stop':
end = time.time()
self.inference_time += (end - self.begin_time)
if self.pair_it == self.total_pairs:
logger.info(
f'Total time spent inferencing pairlist {self.inference_time:.2f} seconds')
if self.inference_time > 0.25 * self.base_tf_seconds:
logger.warning('Inference took over 25/% of the candle time. Reduce pairlist to'
' avoid blinding open trades and degrading performance.')
self.pair_it = 0
self.inference_time = 0
return
# Following methods which are overridden by user made prediction models.
# See freqai/prediction_models/CatboostPredictionModel.py for an example.

99
freqtrade/freqai/prediction_models/BaseClassifierModel.py Normal file
View File

@@ -0,0 +1,99 @@
import logging
from typing import Any, Tuple
import numpy as np
import numpy.typing as npt
import pandas as pd
from pandas import DataFrame
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
from freqtrade.freqai.freqai_interface import IFreqaiModel
logger = logging.getLogger(__name__)
class BaseClassifierModel(IFreqaiModel):
"""
Base class for regression type models (e.g. Catboost, LightGBM, XGboost etc.).
User *must* inherit from this class and set fit() and predict(). See example scripts
such as prediction_models/CatboostPredictionModel.py for guidance.
"""
def train(
self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen
) -> Any:
"""
Filter the training data and train a model to it. Train makes heavy use of the datakitchen
for storing, saving, loading, and analyzing the data.
:param unfiltered_dataframe: Full dataframe for the current training period
:param metadata: pair metadata from strategy.
:return:
:model: Trained model which can be used to inference (self.predict)
"""
logger.info("-------------------- Starting training " f"{pair} --------------------")
# filter the features requested by user in the configuration file and elegantly handle NaNs
features_filtered, labels_filtered = dk.filter_features(
unfiltered_dataframe,
dk.training_features_list,
dk.label_list,
training_filter=True,
)
start_date = unfiltered_dataframe["date"].iloc[0].strftime("%Y-%m-%d")
end_date = unfiltered_dataframe["date"].iloc[-1].strftime("%Y-%m-%d")
logger.info(f"-------------------- Training on data from {start_date} to "
f"{end_date}--------------------")
# split data into train/test data.
data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered)
if not self.freqai_info.get('fit_live_predictions', 0) or not self.live:
dk.fit_labels()
# normalize all data based on train_dataset only
data_dictionary = dk.normalize_data(data_dictionary)
# optional additional data cleaning/analysis
self.data_cleaning_train(dk)
logger.info(
f'Training model on {len(dk.data_dictionary["train_features"].columns)}' " features"
)
logger.info(f'Training model on {len(data_dictionary["train_features"])} data points')
model = self.fit(data_dictionary)
logger.info(f"--------------------done training {pair}--------------------")
return model
def predict(
self, unfiltered_dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = False
) -> Tuple[DataFrame, npt.NDArray[np.int_]]:
"""
Filter the prediction features data and predict with it.
:param: unfiltered_dataframe: Full dataframe for the current backtest period.
:return:
:pred_df: dataframe containing the predictions
:do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove
data (NaNs) or felt uncertain about data (PCA and DI index)
"""
dk.find_features(unfiltered_dataframe)
filtered_dataframe, _ = dk.filter_features(
unfiltered_dataframe, dk.training_features_list, training_filter=False
)
filtered_dataframe = dk.normalize_data_from_metadata(filtered_dataframe)
dk.data_dictionary["prediction_features"] = filtered_dataframe
self.data_cleaning_predict(dk, filtered_dataframe)
predictions = self.model.predict(dk.data_dictionary["prediction_features"])
pred_df = DataFrame(predictions, columns=dk.label_list)
predictions_prob = self.model.predict_proba(dk.data_dictionary["prediction_features"])
pred_df_prob = DataFrame(predictions_prob, columns=self.model.classes_)
pred_df = pd.concat([pred_df, pred_df_prob], axis=1)
return (pred_df, dk.do_predict)

9
freqtrade/freqai/prediction_models/BaseRegressionModel.py
View File

@@ -62,15 +62,6 @@ class BaseRegressionModel(IFreqaiModel):
model = self.fit(data_dictionary)
if pair not in self.dd.historic_predictions:
self.set_initial_historic_predictions(
data_dictionary['train_features'], model, dk, pair)
if self.freqai_info.get('fit_live_predictions_candles', 0) and self.live:
self.fit_live_predictions(dk)
self.dd.save_historic_predictions_to_disk()
logger.info(f"--------------------done training {pair}--------------------")
return model

22
freqtrade/freqai/prediction_models/BaseTensorFlowModel.py
View File

@@ -24,11 +24,11 @@ class BaseTensorFlowModel(IFreqaiModel):
for storing, saving, loading, and analyzing the data.
:param unfiltered_dataframe: Full dataframe for the current training period
:param metadata: pair metadata from strategy.
:returns:
:return:
:model: Trained model which can be used to inference (self.predict)
"""
logger.info("--------------------Starting training " f"{pair} --------------------")
logger.info("-------------------- Starting training " f"{pair} --------------------")
# filter the features requested by user in the configuration file and elegantly handle NaNs
features_filtered, labels_filtered = dk.filter_features(
@@ -38,9 +38,14 @@ class BaseTensorFlowModel(IFreqaiModel):
training_filter=True,
)
start_date = unfiltered_dataframe["date"].iloc[0].strftime("%Y-%m-%d")
end_date = unfiltered_dataframe["date"].iloc[-1].strftime("%Y-%m-%d")
logger.info(f"-------------------- Training on data from {start_date} to "
f"{end_date}--------------------")
# split data into train/test data.
data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered)
if not self.freqai_info.get('fit_live_predictions', 0) or not self.live:
dk.fit_labels()
# normalize all data based on train_dataset only
data_dictionary = dk.normalize_data(data_dictionary)
@@ -54,17 +59,6 @@ class BaseTensorFlowModel(IFreqaiModel):
model = self.fit(data_dictionary)
if pair not in self.dd.historic_predictions:
self.set_initial_historic_predictions(
data_dictionary['train_features'], model, dk, pair)
if self.freqai_info.get('fit_live_predictions_candles', 0) and self.live:
self.fit_live_predictions(dk)
else:
dk.fit_labels()
self.dd.save_historic_predictions_to_disk()
logger.info(f"--------------------done training {pair}--------------------")
return model

4
freqtrade/freqai/prediction_models/CatboostClassifier.py
View File

@@ -3,13 +3,13 @@ from typing import Any, Dict
from catboost import CatBoostClassifier, Pool
from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel
from freqtrade.freqai.prediction_models.BaseClassifierModel import BaseClassifierModel
logger = logging.getLogger(__name__)
class CatboostClassifier(BaseRegressionModel):
class CatboostClassifier(BaseClassifierModel):
"""
User created prediction model. The class needs to override three necessary
functions, predict(), train(), fit(). The class inherits ModelHandler which

4
freqtrade/freqai/prediction_models/LightGBMClassifier.py
View File

@@ -3,13 +3,13 @@ from typing import Any, Dict
from lightgbm import LGBMClassifier
from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel
from freqtrade.freqai.prediction_models.BaseClassifierModel import BaseClassifierModel
logger = logging.getLogger(__name__)
class LightGBMClassifier(BaseRegressionModel):
class LightGBMClassifier(BaseClassifierModel):
"""
User created prediction model. The class needs to override three necessary
functions, predict(), train(), fit(). The class inherits ModelHandler which

6
freqtrade/freqtradebot.py
View File

@@ -1475,12 +1475,6 @@ class FreqtradeBot(LoggingMixin):
ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS, ExitType.LIQUIDATION):
exit_type = 'stoploss'
# if stoploss is on exchange and we are on dry_run mode,
# we consider the sell price stop price
if (self.config['dry_run'] and exit_type == 'stoploss'
and self.strategy.order_types['stoploss_on_exchange']):
limit = trade.stoploss_or_liquidation
# set custom_exit_price if available
proposed_limit_rate = limit
current_profit = trade.calc_profit_ratio(limit)

22
freqtrade/plugins/pairlist/VolumePairList.py
View File

@@ -4,14 +4,14 @@ Volume PairList provider
Provides dynamic pair list based on trade volumes
"""
import logging
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List
import arrow
from cachetools import TTLCache
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
from freqtrade.misc import format_ms_time
from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -158,16 +158,16 @@ class VolumePairList(IPairList):
filtered_tickers: List[Dict[str, Any]] = [{'symbol': k} for k in pairlist]
# get lookback period in ms, for exchange ohlcv fetch
since_ms = int(arrow.utcnow()
.floor('minute')
.shift(minutes=-(self._lookback_period * self._tf_in_min)
- self._tf_in_min)
.int_timestamp) * 1000
since_ms = int(timeframe_to_prev_date(
self._lookback_timeframe,
datetime.now(timezone.utc) + timedelta(
minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min)
).timestamp()) * 1000
to_ms = int(arrow.utcnow()
.floor('minute')
.shift(minutes=-self._tf_in_min)
.int_timestamp) * 1000
to_ms = int(timeframe_to_prev_date(
self._lookback_timeframe,
datetime.now(timezone.utc) - timedelta(minutes=self._tf_in_min)
).timestamp()) * 1000
# todo: utc date output for starting date
self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "

4
freqtrade/plugins/pairlist/pairlist_helpers.py
View File

@@ -1,5 +1,5 @@
import re
from typing import List
from typing import Any, Dict, List
def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
@@ -42,7 +42,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
return result
def dynamic_expand_pairlist(config: dict, markets: list) -> List[str]:
def dynamic_expand_pairlist(config: Dict[str, Any], markets: List[str]) -> List[str]:
expanded_pairs = expand_pairlist(config['pairs'], markets)
if config.get('freqai', {}).get('enabled', False):
corr_pairlist = config['freqai']['feature_parameters']['include_corr_pairlist']

14
freqtrade/rpc/telegram.py
View File

@@ -120,7 +120,8 @@ class Telegram(RPCHandler):
r'/daily$', r'/daily \d+$', r'/profit$', r'/profit \d+',
r'/stats$', r'/count$', r'/locks$', r'/balance$',
r'/stopbuy$', r'/reload_config$', r'/show_config$',
r'/logs$', r'/whitelist$', r'/blacklist$', r'/bl_delete$',
r'/logs$', r'/whitelist$', r'/whitelist(\ssorted|\sbaseonly)+$',
r'/blacklist$', r'/bl_delete$',
r'/weekly$', r'/weekly \d+$', r'/monthly$', r'/monthly \d+$',
r'/forcebuy$', r'/forcelong$', r'/forceshort$',
r'/forcesell$', r'/forceexit$',
@@ -1368,6 +1369,12 @@ class Telegram(RPCHandler):
try:
whitelist = self._rpc._rpc_whitelist()
if context.args:
if "sorted" in context.args:
whitelist['whitelist'] = sorted(whitelist['whitelist'])
if "baseonly" in context.args:
whitelist['whitelist'] = [pair.split("/")[0] for pair in whitelist['whitelist']]
message = f"Using whitelist `{whitelist['method']}` with {whitelist['length']} pairs\n"
message += f"`{', '.join(whitelist['whitelist'])}`"
@@ -1487,7 +1494,8 @@ class Telegram(RPCHandler):
"*/fx <trade_id>|all:* `Alias to /forceexit`\n"
f"{force_enter_text if self._config.get('force_entry_enable', False) else ''}"
"*/delete <trade_id>:* `Instantly delete the given trade in the database`\n"
"*/whitelist:* `Show current whitelist` \n"
"*/whitelist [sorted] [baseonly]:* `Show current whitelist. Optionally in "
"order and/or only displaying the base currency of each pairing.`\n"
"*/blacklist [pair]:* `Show current blacklist, or adds one or more pairs "
"to the blacklist.` \n"
"*/blacklist_delete [pairs]| /bl_delete [pairs]:* "
@@ -1524,7 +1532,7 @@ class Telegram(RPCHandler):
"*/weekly <n>:* `Shows statistics per week, over the last n weeks`\n"
"*/monthly <n>:* `Shows statistics per month, over the last n months`\n"
"*/stats:* `Shows Wins / losses by Sell reason as well as "
"Avg. holding durationsfor buys and sells.`\n"
"Avg. holding durations for buys and sells.`\n"
"*/help:* `This help message`\n"
"*/version:* `Show version`"
)

3
freqtrade/strategy/interface.py
View File

@@ -157,7 +157,8 @@ class IStrategy(ABC, HyperStrategyMixin):
class DummyClass():
def start(self, *args, **kwargs):
raise OperationalException(
'freqAI is not enabled. Please enable it in your config to use this strategy.')
'freqAI is not enabled. '
'Please enable it in your config to use this strategy.')
self.freqai = DummyClass() # type: ignore
def ft_bot_start(self, **kwargs) -> None:

177
freqtrade/templates/FreqaiExampleStrategy.py
View File

@@ -82,99 +82,98 @@ class FreqaiExampleStrategy(IStrategy):
coin = pair.split('/')[0]
with self.freqai.lock:
if informative is None:
informative = self.dp.get_pair_dataframe(pair, tf)
if informative is None:
informative = self.dp.get_pair_dataframe(pair, tf)
# first loop is automatically duplicating indicators for time periods
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
# first loop is automatically duplicating indicators for time periods
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t)
informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t)
informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
t = int(t)
informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t)
informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
bollinger = qtpylib.bollinger_bands(
qtpylib.typical_price(informative), window=t, stds=2.2
)
informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"]
informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"]
informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"]
bollinger = qtpylib.bollinger_bands(
qtpylib.typical_price(informative), window=t, stds=2.2
)
informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"]
informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"]
informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"]
informative[f"%-{coin}bb_width-period_{t}"] = (
informative[f"{coin}bb_upperband-period_{t}"]
- informative[f"{coin}bb_lowerband-period_{t}"]
) / informative[f"{coin}bb_middleband-period_{t}"]
informative[f"%-{coin}close-bb_lower-period_{t}"] = (
informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"]
)
informative[f"%-{coin}bb_width-period_{t}"] = (
informative[f"{coin}bb_upperband-period_{t}"]
- informative[f"{coin}bb_lowerband-period_{t}"]
) / informative[f"{coin}bb_middleband-period_{t}"]
informative[f"%-{coin}close-bb_lower-period_{t}"] = (
informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"]
)
informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t)
informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t)
informative[f"%-{coin}relative_volume-period_{t}"] = (
informative["volume"] / informative["volume"].rolling(t).mean()
)
informative[f"%-{coin}relative_volume-period_{t}"] = (
informative["volume"] / informative["volume"].rolling(t).mean()
)
informative[f"%-{coin}pct-change"] = informative["close"].pct_change()
informative[f"%-{coin}raw_volume"] = informative["volume"]
informative[f"%-{coin}raw_price"] = informative["close"]
informative[f"%-{coin}pct-change"] = informative["close"].pct_change()
informative[f"%-{coin}raw_volume"] = informative["volume"]
informative[f"%-{coin}raw_price"] = informative["close"]
indicators = [col for col in informative if col.startswith("%")]
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
indicators = [col for col in informative if col.startswith("%")]
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True)
skip_columns = [
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True)
skip_columns = [
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
# Add generalized indicators here (because in live, it will call this
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
# Add generalized indicators here (because in live, it will call this
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
# user adds targets here by prepending them with &- (see convention below)
df["&-s_close"] = (
df["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.mean()
/ df["close"]
- 1
)
# user adds targets here by prepending them with &- (see convention below)
df["&-s_close"] = (
df["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.mean()
/ df["close"]
- 1
)
# Classifiers are typically set up with strings as targets:
# df['&s-up_or_down'] = np.where( df["close"].shift(-100) >
# df["close"], 'up', 'down')
# Classifiers are typically set up with strings as targets:
# df['&s-up_or_down'] = np.where( df["close"].shift(-100) >
# df["close"], 'up', 'down')
# If user wishes to use multiple targets, they can add more by
# appending more columns with '&'. User should keep in mind that multi targets
# requires a multioutput prediction model such as
# templates/CatboostPredictionMultiModel.py,
# If user wishes to use multiple targets, they can add more by
# appending more columns with '&'. User should keep in mind that multi targets
# requires a multioutput prediction model such as
# templates/CatboostPredictionMultiModel.py,
# df["&-s_range"] = (
# df["close"]
# .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
# .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
# .max()
# -
# df["close"]
# .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
# .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
# .min()
# )
# df["&-s_range"] = (
# df["close"]
# .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
# .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
# .max()
# -
# df["close"]
# .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
# .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
# .min()
# )
return df
@@ -252,12 +251,11 @@ class FreqaiExampleStrategy(IStrategy):
"prediction" + entry_tag not in pair_dict[pair]
or pair_dict[pair]['extras']["prediction" + entry_tag] == 0
):
with self.freqai.lock:
pair_dict[pair]['extras']["prediction" + entry_tag] = abs(trade_candle["&-s_close"])
if not follow_mode:
self.freqai.dd.save_drawer_to_disk()
else:
self.freqai.dd.save_follower_dict_to_disk()
pair_dict[pair]['extras']["prediction" + entry_tag] = abs(trade_candle["&-s_close"])
if not follow_mode:
self.freqai.dd.save_drawer_to_disk()
else:
self.freqai.dd.save_follower_dict_to_disk()
roi_price = pair_dict[pair]['extras']["prediction" + entry_tag]
roi_time = self.max_roi_time_long.value
@@ -296,12 +294,11 @@ class FreqaiExampleStrategy(IStrategy):
else:
pair_dict = self.freqai.dd.follower_dict
with self.freqai.lock:
pair_dict[pair]['extras']["prediction" + entry_tag] = 0
if not follow_mode:
self.freqai.dd.save_drawer_to_disk()
else:
self.freqai.dd.save_follower_dict_to_disk()
pair_dict[pair]['extras']["prediction" + entry_tag] = 0
if not follow_mode:
self.freqai.dd.save_drawer_to_disk()
else:
self.freqai.dd.save_follower_dict_to_disk()
return True