adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into add-xgboostclassifier

Browse Source
This commit is contained in:
Emre
2022-09-10 23:59:11 +03:00
committed by GitHub
parent 60eb02bb62 075748b21a
commit 330d7068ab
24 changed files with 254 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/commands/db_commands.py
View File

@@ -4,7 +4,7 @@ from typing import Any, Dict
from sqlalchemy import func
from freqtrade.configuration.config_setup import setup_utils_configuration
from freqtrade.enums.runmode import RunMode
from freqtrade.enums import RunMode
logger = logging.getLogger(__name__)

9
freqtrade/configuration/config_validation.py
View File

@@ -84,6 +84,7 @@ def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False)
_validate_protections(conf)
_validate_unlimited_amount(conf)
_validate_ask_orderbook(conf)
_validate_freqai_hyperopt(conf)
validate_migrated_strategy_settings(conf)
# validate configuration before returning
@@ -323,6 +324,14 @@ def _validate_pricing_rules(conf: Dict[str, Any]) -> None:
del conf['ask_strategy']
def _validate_freqai_hyperopt(conf: Dict[str, Any]) -> None:
freqai_enabled = conf.get('freqai', {}).get('enabled', False)
analyze_per_epoch = conf.get('analyze_per_epoch', False)
if analyze_per_epoch and freqai_enabled:
raise OperationalException(
'Using analyze-per-epoch parameter is not supported with a FreqAI strategy.')
def _strategy_settings(conf: Dict[str, Any]) -> None:
process_deprecated_setting(conf, None, 'use_sell_signal', None, 'use_exit_signal')

8
freqtrade/data/history/history_utils.py
View File

@@ -228,9 +228,9 @@ def _download_pair_history(pair: str, *,
)
logger.debug("Current Start: %s",
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
f"{data.iloc[0]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None')
logger.debug("Current End: %s",
f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
f"{data.iloc[-1]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None')
# Default since_ms to 30 days if nothing is given
new_data = exchange.get_historic_ohlcv(pair=pair,
@@ -254,9 +254,9 @@ def _download_pair_history(pair: str, *,
fill_missing=False, drop_incomplete=False)
logger.debug("New Start: %s",
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
f"{data.iloc[0]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None')
logger.debug("New End: %s",
f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
f"{data.iloc[-1]['date']:DATETIME_PRINT_FORMAT}" if not data.empty else 'None')
data_handler.ohlcv_store(pair, timeframe, data=data, candle_type=candle_type)
return True

3
freqtrade/exchange/okx.py
View File

@@ -4,8 +4,7 @@ from typing import Dict, List, Optional, Tuple
import ccxt
from freqtrade.constants import BuySell
from freqtrade.enums import MarginMode, TradingMode
from freqtrade.enums.candletype import CandleType
from freqtrade.enums import CandleType, MarginMode, TradingMode
from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError
from freqtrade.exchange import Exchange, date_minus_candles
from freqtrade.exchange.common import retrier

10
freqtrade/freqai/base_models/FreqaiMultiOutputRegressor.py
View File

@@ -36,9 +36,6 @@ class FreqaiMultiOutputRegressor(MultiOutputRegressor):
y = self._validate_data(X="no_validation", y=y, multi_output=True)
# if is_classifier(self):
# check_classification_targets(y)
if y.ndim == 1:
raise ValueError(
"y must have at least two dimensions for "
@@ -50,19 +47,12 @@ class FreqaiMultiOutputRegressor(MultiOutputRegressor):
):
raise ValueError("Underlying estimator does not support sample weights.")
# fit_params_validated = _check_fit_params(X, fit_params)
if not fit_params:
fit_params = [None] * y.shape[1]
# if not init_models:
# init_models = [None] * y.shape[1]
self.estimators_ = Parallel(n_jobs=self.n_jobs)(
delayed(_fit_estimator)(
self.estimator, X, y[:, i], sample_weight, **fit_params[i]
# init_model=init_models[i], eval_set=eval_sets[i],
# **fit_params_validated
)
for i in range(y.shape[1])
)

51
freqtrade/freqai/data_kitchen.py
View File

@@ -184,7 +184,7 @@ class FreqaiDataKitchen:
def filter_features(
self,
unfiltered_dataframe: DataFrame,
unfiltered_df: DataFrame,
training_feature_list: List,
label_list: List = list(),
training_filter: bool = True,
@@ -195,31 +195,35 @@ class FreqaiDataKitchen:
0s in the prediction dataset. However, prediction dataset do_predict will reflect any
row that had a NaN and will shield user from that prediction.
:params:
:unfiltered_dataframe: the full dataframe for the present training period
:unfiltered_df: the full dataframe for the present training period
:training_feature_list: list, the training feature list constructed by
self.build_feature_list() according to user specified parameters in the configuration file.
:labels: the labels for the dataset
:training_filter: boolean which lets the function know if it is training data or
prediction data to be filtered.
:returns:
:filtered_dataframe: dataframe cleaned of NaNs and only containing the user
:filtered_df: dataframe cleaned of NaNs and only containing the user
requested feature set.
:labels: labels cleaned of NaNs.
"""
filtered_dataframe = unfiltered_dataframe.filter(training_feature_list, axis=1)
filtered_dataframe = filtered_dataframe.replace([np.inf, -np.inf], np.nan)
filtered_df = unfiltered_df.filter(training_feature_list, axis=1)
filtered_df = filtered_df.replace([np.inf, -np.inf], np.nan)
drop_index = pd.isnull(filtered_dataframe).any(1) # get the rows that have NaNs,
drop_index = pd.isnull(filtered_df).any(1) # get the rows that have NaNs,
drop_index = drop_index.replace(True, 1).replace(False, 0) # pep8 requirement.
if (training_filter):
const_cols = list((filtered_df.nunique() == 1).loc[lambda x: x].index)
if const_cols:
filtered_df = filtered_df.filter(filtered_df.columns.difference(const_cols))
logger.warning(f"Removed features {const_cols} with constant values.")
# we don't care about total row number (total no. datapoints) in training, we only care
# about removing any row with NaNs
# if labels has multiple columns (user wants to train multiple modelEs), we detect here
labels = unfiltered_dataframe.filter(label_list, axis=1)
labels = unfiltered_df.filter(label_list, axis=1)
drop_index_labels = pd.isnull(labels).any(1)
drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0)
dates = unfiltered_dataframe['date']
filtered_dataframe = filtered_dataframe[
dates = unfiltered_df['date']
filtered_df = filtered_df[
(drop_index == 0) & (drop_index_labels == 0)
] # dropping values
labels = labels[
@@ -229,13 +233,13 @@ class FreqaiDataKitchen:
(drop_index == 0) & (drop_index_labels == 0)
]
logger.info(
f"dropped {len(unfiltered_dataframe) - len(filtered_dataframe)} training points"
f" due to NaNs in populated dataset {len(unfiltered_dataframe)}."
f"dropped {len(unfiltered_df) - len(filtered_df)} training points"
f" due to NaNs in populated dataset {len(unfiltered_df)}."
)
if (1 - len(filtered_dataframe) / len(unfiltered_dataframe)) > 0.1 and self.live:
worst_indicator = str(unfiltered_dataframe.count().idxmin())
if (1 - len(filtered_df) / len(unfiltered_df)) > 0.1 and self.live:
worst_indicator = str(unfiltered_df.count().idxmin())
logger.warning(
f" {(1 - len(filtered_dataframe)/len(unfiltered_dataframe)) * 100:.0f} percent "
f" {(1 - len(filtered_df)/len(unfiltered_df)) * 100:.0f} percent "
" of training data dropped due to NaNs, model may perform inconsistent "
f"with expectations. Verify {worst_indicator}"
)
@@ -244,9 +248,9 @@ class FreqaiDataKitchen:
else:
# we are backtesting so we need to preserve row number to send back to strategy,
# so now we use do_predict to avoid any prediction based on a NaN
drop_index = pd.isnull(filtered_dataframe).any(1)
drop_index = pd.isnull(filtered_df).any(1)
self.data["filter_drop_index_prediction"] = drop_index
filtered_dataframe.fillna(0, inplace=True)
filtered_df.fillna(0, inplace=True)
# replacing all NaNs with zeros to avoid issues in 'prediction', but any prediction
# that was based on a single NaN is ultimately protected from buys with do_predict
drop_index = ~drop_index
@@ -255,11 +259,11 @@ class FreqaiDataKitchen:
logger.info(
"dropped %s of %s prediction data points due to NaNs.",
len(self.do_predict) - self.do_predict.sum(),
len(filtered_dataframe),
len(filtered_df),
)
labels = []
return filtered_dataframe, labels
return filtered_df, labels
def build_data_dictionary(
self,
@@ -466,10 +470,17 @@ class FreqaiDataKitchen:
) -> DataFrame:
"""
Function which takes the backtesting time range and
remove training data from dataframe
remove training data from dataframe, keeping only the
startup_candle_count candles
"""
startup_candle_count = self.config.get('startup_candle_count', 0)
tf = self.config['timeframe']
tr = self.config["timerange"]
backtesting_timerange = TimeRange.parse_timerange(tr)
if startup_candle_count > 0 and backtesting_timerange:
backtesting_timerange.subtract_start(timeframe_to_seconds(tf) * startup_candle_count)
start = datetime.fromtimestamp(backtesting_timerange.startts, tz=timezone.utc)
df = self.return_dataframe
df = df.loc[df["date"] >= start, :]
@@ -1215,7 +1226,6 @@ class FreqaiDataKitchen:
def save_backtesting_prediction(
self, append_df: DataFrame
) -> None:
"""
Save prediction dataframe from backtesting to h5 file format
:param append_df: dataframe for backtesting period
@@ -1229,7 +1239,6 @@ class FreqaiDataKitchen:
def get_backtesting_prediction(
self
) -> DataFrame:
"""
Get prediction dataframe from h5 file format
"""

11
freqtrade/freqai/freqai_interface.py
View File

@@ -14,6 +14,7 @@ from numpy.typing import NDArray
from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.enums import RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_seconds
@@ -92,6 +93,12 @@ class IFreqaiModel(ABC):
self._threads: List[threading.Thread] = []
self._stop_event = threading.Event()
def __getstate__(self):
"""
Return an empty state to be pickled in hyperopt
"""
return ({})
def assert_config(self, config: Dict[str, Any]) -> None:
if not config.get("freqai", {}):
@@ -233,10 +240,10 @@ class IFreqaiModel(ABC):
trained_timestamp = tr_train
tr_train_startts_str = datetime.fromtimestamp(
tr_train.startts,
tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
tr_train_stopts_str = datetime.fromtimestamp(
tr_train.stopts,
tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
logger.info(
f"Training {metadata['pair']}, {self.pair_it}/{self.total_pairs} pairs"
f" from {tr_train_startts_str} to {tr_train_stopts_str}, {train_it}/{total_trains} "

3
freqtrade/freqai/prediction_models/CatboostRegressorMultiTarget.py
View File

@@ -60,6 +60,9 @@ class CatboostRegressorMultiTarget(BaseRegressionModel):
{'eval_set': eval_sets[i], 'init_model': init_models[i]})
model = FreqaiMultiOutputRegressor(estimator=cbr)
thread_training = self.freqai_info.get('multitarget_parallel_training', False)
if thread_training:
model.n_jobs = y.shape[1]
model.fit(X=X, y=y, sample_weight=sample_weight, fit_params=fit_params)
return model

6
freqtrade/freqai/prediction_models/LightGBMRegressorMultiTarget.py
View File

@@ -56,9 +56,9 @@ class LightGBMRegressorMultiTarget(BaseRegressionModel):
'init_model': init_models[i]})
model = FreqaiMultiOutputRegressor(estimator=lgb)
thread_training = self.freqai_info.get('multitarget_parallel_training', False)
if thread_training:
model.n_jobs = y.shape[1]
model.fit(X=X, y=y, sample_weight=sample_weight, fit_params=fit_params)
# model = FreqaiMultiOutputRegressor(estimator=lgb)
# model.fit(X=X, y=y, sample_weight=sample_weight, init_models=init_models,
# eval_sets=eval_sets, eval_sample_weight=eval_weights)
return model

3
freqtrade/freqai/prediction_models/XGBoostRegressorMultiTarget.py
View File

@@ -55,6 +55,9 @@ class XGBoostRegressorMultiTarget(BaseRegressionModel):
'xgb_model': init_models[i]})
model = FreqaiMultiOutputRegressor(estimator=xgb)
thread_training = self.freqai_info.get('multitarget_parallel_training', False)
if thread_training:
model.n_jobs = y.shape[1]
model.fit(X=X, y=y, sample_weight=sample_weight, fit_params=fit_params)
return model

5
freqtrade/optimize/optimize_reports.py
View File

@@ -75,7 +75,8 @@ def _get_line_floatfmt(stake_currency: str) -> List[str]:
'.2f', 'd', 's', 's']
def _get_line_header(first_column: str, stake_currency: str, direction: str = 'Buys') -> List[str]:
def _get_line_header(first_column: str, stake_currency: str,
direction: str = 'Entries') -> List[str]:
"""
Generate header lines (goes in line with _generate_result_line())
"""
@@ -642,7 +643,7 @@ def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_curr
if (tag_type == "enter_tag"):
headers = _get_line_header("TAG", stake_currency)
else:
headers = _get_line_header("TAG", stake_currency, 'Sells')
headers = _get_line_header("TAG", stake_currency, 'Exits')
floatfmt = _get_line_floatfmt(stake_currency)
output = [
[

2
freqtrade/rpc/discord.py
View File

@@ -1,7 +1,7 @@
import logging
from typing import Any, Dict
from freqtrade.enums.rpcmessagetype import RPCMessageType
from freqtrade.enums import RPCMessageType
from freqtrade.rpc import RPC
from freqtrade.rpc.webhook import Webhook

5
freqtrade/strategy/interface.py
View File

@@ -12,9 +12,8 @@ from pandas import DataFrame
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, SignalDirection, SignalTagType,
SignalType, TradingMode)
from freqtrade.enums.runmode import RunMode
from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, RunMode, SignalDirection,
SignalTagType, SignalType, TradingMode)
from freqtrade.exceptions import OperationalException, StrategyError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date, timeframe_to_seconds
from freqtrade.persistence import Order, PairLocks, Trade

2
freqtrade/strategy/parameters.py
View File

@@ -7,7 +7,7 @@ from abc import ABC, abstractmethod
from contextlib import suppress
from typing import Any, Optional, Sequence, Union
from freqtrade.enums.hyperoptstate import HyperoptState
from freqtrade.enums import HyperoptState
from freqtrade.optimize.hyperopt_tools import HyperoptStateContainer

113
freqtrade/templates/FreqaiExampleStrategy.py
View File

@@ -6,9 +6,7 @@ import talib.abstract as ta
from pandas import DataFrame
from technical import qtpylib
from freqtrade.exchange import timeframe_to_prev_date
from freqtrade.persistence import Trade
from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy, merge_informative_pair
from freqtrade.strategy import CategoricalParameter, IStrategy, merge_informative_pair
logger = logging.getLogger(__name__)
@@ -31,9 +29,6 @@ class FreqaiExampleStrategy(IStrategy):
"main_plot": {},
"subplots": {
"prediction": {"prediction": {"color": "blue"}},
"target_roi": {
"target_roi": {"color": "brown"},
},
"do_predict": {
"do_predict": {"color": "brown"},
},
@@ -47,10 +42,10 @@ class FreqaiExampleStrategy(IStrategy):
startup_candle_count: int = 40
can_short = False
linear_roi_offset = DecimalParameter(
0.00, 0.02, default=0.005, space="sell", optimize=False, load=True
)
max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True)
std_dev_multiplier_buy = CategoricalParameter(
[0.75, 1, 1.25, 1.5, 1.75], default=1.25, space="buy", optimize=True)
std_dev_multiplier_sell = CategoricalParameter(
[0.1, 0.25, 0.4], space="sell", default=0.2, optimize=True)
def informative_pairs(self):
whitelist_pairs = self.dp.current_whitelist()
@@ -187,21 +182,26 @@ class FreqaiExampleStrategy(IStrategy):
# `populate_any_indicators()` for each training period.
dataframe = self.freqai.start(dataframe, metadata, self)
dataframe["target_roi"] = dataframe["&-s_close_mean"] + dataframe["&-s_close_std"] * 1.25
dataframe["sell_roi"] = dataframe["&-s_close_mean"] - dataframe["&-s_close_std"] * 1.25
for val in self.std_dev_multiplier_buy.range:
dataframe[f'target_roi_{val}'] = dataframe["&-s_close_mean"] + \
dataframe["&-s_close_std"] * val
for val in self.std_dev_multiplier_sell.range:
dataframe[f'sell_roi_{val}'] = dataframe["&-s_close_mean"] - \
dataframe["&-s_close_std"] * val
return dataframe
def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame:
enter_long_conditions = [df["do_predict"] == 1, df["&-s_close"] > df["target_roi"]]
enter_long_conditions = [df["do_predict"] == 1, df["&-s_close"]
> df[f"target_roi_{self.std_dev_multiplier_buy.value}"]]
if enter_long_conditions:
df.loc[
reduce(lambda x, y: x & y, enter_long_conditions), ["enter_long", "enter_tag"]
] = (1, "long")
enter_short_conditions = [df["do_predict"] == 1, df["&-s_close"] < df["sell_roi"]]
enter_short_conditions = [df["do_predict"] == 1, df["&-s_close"]
< df[f"sell_roi_{self.std_dev_multiplier_sell.value}"]]
if enter_short_conditions:
df.loc[
@@ -211,11 +211,13 @@ class FreqaiExampleStrategy(IStrategy):
return df
def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame:
exit_long_conditions = [df["do_predict"] == 1, df["&-s_close"] < df["sell_roi"] * 0.25]
exit_long_conditions = [df["do_predict"] == 1, df["&-s_close"] <
df[f"sell_roi_{self.std_dev_multiplier_sell.value}"] * 0.25]
if exit_long_conditions:
df.loc[reduce(lambda x, y: x & y, exit_long_conditions), "exit_long"] = 1
exit_short_conditions = [df["do_predict"] == 1, df["&-s_close"] > df["target_roi"] * 0.25]
exit_short_conditions = [df["do_predict"] == 1, df["&-s_close"] >
df[f"target_roi_{self.std_dev_multiplier_buy.value}"] * 0.25]
if exit_short_conditions:
df.loc[reduce(lambda x, y: x & y, exit_short_conditions), "exit_short"] = 1
@@ -224,83 +226,6 @@ class FreqaiExampleStrategy(IStrategy):
def get_ticker_indicator(self):
return int(self.config["timeframe"][:-1])
def custom_exit(
self, pair: str, trade: Trade, current_time, current_rate, current_profit, **kwargs
):
dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe)
trade_date = timeframe_to_prev_date(self.config["timeframe"], trade.open_date_utc)
trade_candle = dataframe.loc[(dataframe["date"] == trade_date)]
if trade_candle.empty:
return None
trade_candle = trade_candle.squeeze()
follow_mode = self.config.get("freqai", {}).get("follow_mode", False)
if not follow_mode:
pair_dict = self.freqai.dd.pair_dict
else:
pair_dict = self.freqai.dd.follower_dict
entry_tag = trade.enter_tag
if (
"prediction" + entry_tag not in pair_dict[pair]
or pair_dict[pair]['extras']["prediction" + entry_tag] == 0
):
pair_dict[pair]['extras']["prediction" + entry_tag] = abs(trade_candle["&-s_close"])
if not follow_mode:
self.freqai.dd.save_drawer_to_disk()
else:
self.freqai.dd.save_follower_dict_to_disk()
roi_price = pair_dict[pair]['extras']["prediction" + entry_tag]
roi_time = self.max_roi_time_long.value
roi_decay = roi_price * (
1 - ((current_time - trade.open_date_utc).seconds) / (roi_time * 60)
)
if roi_decay < 0:
roi_decay = self.linear_roi_offset.value
else:
roi_decay += self.linear_roi_offset.value
if current_profit > roi_decay:
return "roi_custom_win"
if current_profit < -roi_decay:
return "roi_custom_loss"
def confirm_trade_exit(
self,
pair: str,
trade: Trade,
order_type: str,
amount: float,
rate: float,
time_in_force: str,
exit_reason: str,
current_time,
**kwargs,
) -> bool:
entry_tag = trade.enter_tag
follow_mode = self.config.get("freqai", {}).get("follow_mode", False)
if not follow_mode:
pair_dict = self.freqai.dd.pair_dict
else:
pair_dict = self.freqai.dd.follower_dict
pair_dict[pair]['extras']["prediction" + entry_tag] = 0
if not follow_mode:
self.freqai.dd.save_drawer_to_disk()
else:
self.freqai.dd.save_follower_dict_to_disk()
return True
def confirm_trade_entry(
self,
pair: str,