adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove follow mode in favor of producer consumer

Browse Source
This commit is contained in:
robcaulk 2023-02-02 11:40:23 +01:00
parent 1d6738778b
commit ccb4efbe88
3 changed files with 17 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
freqtrade/freqai/data_drawer.py
View File

@ -59,7 +59,7 @@ class FreqaiDataDrawer:
Juha Nykänen @suikula, Wagner Costa @wagnercosta, Johan Vlugt @Jooopieeert Juha Nykänen @suikula, Wagner Costa @wagnercosta, Johan Vlugt @Jooopieeert
""" """
def __init__(self, full_path: Path, config: Config, follow_mode: bool = False): def __init__(self, full_path: Path, config: Config):
self.config = config self.config = config
self.freqai_info = config.get("freqai", {}) self.freqai_info = config.get("freqai", {})
@ -84,9 +84,6 @@ class FreqaiDataDrawer:
self.pair_dictionary_path = Path(self.full_path / "pair_dictionary.json") self.pair_dictionary_path = Path(self.full_path / "pair_dictionary.json")
self.global_metadata_path = Path(self.full_path / "global_metadata.json") self.global_metadata_path = Path(self.full_path / "global_metadata.json")
self.metric_tracker_path = Path(self.full_path / "metric_tracker.json") self.metric_tracker_path = Path(self.full_path / "metric_tracker.json")
self.follow_mode = follow_mode
if follow_mode:
self.create_follower_dict()
self.load_drawer_from_disk() self.load_drawer_from_disk()
self.load_historic_predictions_from_disk() self.load_historic_predictions_from_disk()
self.metric_tracker: Dict[str, Dict[str, Dict[str, list]]] = {} self.metric_tracker: Dict[str, Dict[str, Dict[str, list]]] = {}
@ -149,13 +146,8 @@ class FreqaiDataDrawer:
if exists: if exists:
with open(self.pair_dictionary_path, "r") as fp: with open(self.pair_dictionary_path, "r") as fp:
self.pair_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE) self.pair_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
elif not self.follow_mode:
logger.info("Could not find existing datadrawer, starting from scratch")
else: else:
logger.warning( logger.info("Could not find existing datadrawer, starting from scratch")
f"Follower could not find pair_dictionary at {self.full_path} "
"sending null values back to strategy"
)
def load_metric_tracker_from_disk(self): def load_metric_tracker_from_disk(self):
""" """
@ -193,13 +185,8 @@ class FreqaiDataDrawer:
self.historic_predictions = cloudpickle.load(fp) self.historic_predictions = cloudpickle.load(fp)
logger.warning('FreqAI successfully loaded the backup historical predictions file.') logger.warning('FreqAI successfully loaded the backup historical predictions file.')
elif not self.follow_mode:
logger.info("Could not find existing historic_predictions, starting from scratch")
else: else:
logger.warning( logger.info("Could not find existing historic_predictions, starting from scratch")
f"Follower could not find historic predictions at {self.full_path} "
"sending null values back to strategy"
)
return exists return exists
@ -248,23 +235,6 @@ class FreqaiDataDrawer:
rapidjson.dump(metadata, fp, default=self.np_encoder, rapidjson.dump(metadata, fp, default=self.np_encoder,
number_mode=rapidjson.NM_NATIVE) number_mode=rapidjson.NM_NATIVE)
def create_follower_dict(self):
"""
Create or dictionary for each follower to maintain unique persistent prediction targets
"""
whitelist_pairs = self.config.get("exchange", {}).get("pair_whitelist")
exists = self.follower_dict_path.is_file()
if exists:
logger.info("Found an existing follower dictionary")
for pair in whitelist_pairs:
self.follower_dict[pair] = {}
self.save_follower_dict_to_disk()
def np_encoder(self, object): def np_encoder(self, object):
if isinstance(object, np.generic): if isinstance(object, np.generic):
return object.item() return object.item()
@ -282,27 +252,17 @@ class FreqaiDataDrawer:
""" """
pair_dict = self.pair_dict.get(pair) pair_dict = self.pair_dict.get(pair)
data_path_set = self.pair_dict.get(pair, self.empty_pair_dict).get("data_path", "") # data_path_set = self.pair_dict.get(pair, self.empty_pair_dict).get("data_path", "")
return_null_array = False return_null_array = False
if pair_dict: if pair_dict:
model_filename = pair_dict["model_filename"] model_filename = pair_dict["model_filename"]
trained_timestamp = pair_dict["trained_timestamp"] trained_timestamp = pair_dict["trained_timestamp"]
elif not self.follow_mode: else:
self.pair_dict[pair] = self.empty_pair_dict.copy() self.pair_dict[pair] = self.empty_pair_dict.copy()
model_filename = "" model_filename = ""
trained_timestamp = 0 trained_timestamp = 0
if not data_path_set and self.follow_mode:
logger.warning(
f"Follower could not find current pair {pair} in "
f"pair_dictionary at path {self.full_path}, sending null values "
"back to strategy."
)
trained_timestamp = 0
model_filename = ''
return_null_array = True
return model_filename, trained_timestamp, return_null_array return model_filename, trained_timestamp, return_null_array
def set_pair_dict_info(self, metadata: dict) -> None: def set_pair_dict_info(self, metadata: dict) -> None:
@ -311,7 +271,6 @@ class FreqaiDataDrawer:
return return
else: else:
self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy() self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy()
return return
def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None: def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None:

43
freqtrade/freqai/freqai_interface.py
View File

@ -66,12 +66,11 @@ class IFreqaiModel(ABC):
self.retrain = False self.retrain = False
self.first = True self.first = True
self.set_full_path() self.set_full_path()
self.follow_mode: bool = self.freqai_info.get("follow_mode", False)
self.save_backtest_models: bool = self.freqai_info.get("save_backtest_models", True) self.save_backtest_models: bool = self.freqai_info.get("save_backtest_models", True)
if self.save_backtest_models: if self.save_backtest_models:
logger.info('Backtesting module configured to save all models.') logger.info('Backtesting module configured to save all models.')
self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode) self.dd = FreqaiDataDrawer(Path(self.full_path), self.config)
# set current candle to arbitrary historical date # set current candle to arbitrary historical date
self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc) self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc)
self.dd.current_candle = self.current_candle self.dd.current_candle = self.current_candle
@ -153,7 +152,7 @@ class IFreqaiModel(ABC):
# (backtest window, i.e. window immediately following the training window). # (backtest window, i.e. window immediately following the training window).
# FreqAI slides the window and sequentially builds the backtesting results before returning # FreqAI slides the window and sequentially builds the backtesting results before returning
# the concatenated results for the full backtesting period back to the strategy. # the concatenated results for the full backtesting period back to the strategy.
elif not self.follow_mode: else:
self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"]) self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
if not self.config.get("freqai_backtest_live_models", False): if not self.config.get("freqai_backtest_live_models", False):
logger.info(f"Training {len(self.dk.training_timeranges)} timeranges") logger.info(f"Training {len(self.dk.training_timeranges)} timeranges")
@ -379,46 +378,28 @@ class IFreqaiModel(ABC):
:returns: :returns:
dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only
""" """
# update follower
if self.follow_mode:
self.dd.update_follower_metadata()
# get the model metadata associated with the current pair # get the model metadata associated with the current pair
(_, trained_timestamp, return_null_array) = self.dd.get_pair_dict_info(metadata["pair"]) (_, trained_timestamp, return_null_array) = self.dd.get_pair_dict_info(metadata["pair"])
# if the metadata doesn't exist, the follower returns null arrays to strategy
if self.follow_mode and return_null_array:
logger.info("Returning null array from follower to strategy")
self.dd.return_null_values_to_strategy(dataframe, dk)
return dk
# append the historic data once per round # append the historic data once per round
if self.dd.historic_data: if self.dd.historic_data:
self.dd.update_historic_data(strategy, dk) self.dd.update_historic_data(strategy, dk)
logger.debug(f'Updating historic data on pair {metadata["pair"]}') logger.debug(f'Updating historic data on pair {metadata["pair"]}')
self.track_current_candle() self.track_current_candle()
if not self.follow_mode: (_, new_trained_timerange, data_load_timerange) = dk.check_if_new_training_required(
trained_timestamp
)
dk.set_paths(metadata["pair"], new_trained_timerange.stopts)
(_, new_trained_timerange, data_load_timerange) = dk.check_if_new_training_required( # load candle history into memory if it is not yet.
trained_timestamp if not self.dd.historic_data:
) self.dd.load_all_pair_histories(data_load_timerange, dk)
dk.set_paths(metadata["pair"], new_trained_timerange.stopts)
# load candle history into memory if it is not yet. if not self.scanning:
if not self.dd.historic_data: self.scanning = True
self.dd.load_all_pair_histories(data_load_timerange, dk) self.start_scanning(strategy)
if not self.scanning:
self.scanning = True
self.start_scanning(strategy)
elif self.follow_mode:
dk.set_paths(metadata["pair"], trained_timestamp)
logger.info(
"FreqAI instance set to follow_mode, finding existing pair "
f"using { self.identifier }"
)
# load the model and associated data into the data kitchen # load the model and associated data into the data kitchen
self.model = self.dd.load_data(metadata["pair"], dk) self.model = self.dd.load_data(metadata["pair"], dk)

51
tests/freqai/test_freqai_interface.py
View File

@ -376,57 +376,6 @@ def test_backtesting_fit_live_predictions(mocker, freqai_conf, caplog):
shutil.rmtree(Path(freqai.dk.full_path)) shutil.rmtree(Path(freqai.dk.full_path))
def test_follow_mode(mocker, freqai_conf):
freqai_conf.update({"timerange": "20180110-20180130"})
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.live = True
freqai.dk = FreqaiDataKitchen(freqai_conf)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
metadata = {"pair": "ADA/BTC"}
freqai.dd.set_pair_dict_info(metadata)
data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
new_timerange = TimeRange.parse_timerange("20180120-20180130")
freqai.extract_data_and_train_model(
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").is_file()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").is_file()
# start the follower and ask it to predict on existing files
freqai_conf.get("freqai", {}).update({"follow_mode": "true"})
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.live = True
freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.live)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
df = strategy.dp.get_pair_dataframe('ADA/BTC', '5m')
freqai.dk.pair = "ADA/BTC"
freqai.start_live(df, metadata, strategy, freqai.dk)
assert len(freqai.dk.return_dataframe.index) == 5702
shutil.rmtree(Path(freqai.dk.full_path))
def test_principal_component_analysis(mocker, freqai_conf): def test_principal_component_analysis(mocker, freqai_conf):
freqai_conf.update({"timerange": "20180110-20180130"}) freqai_conf.update({"timerange": "20180110-20180130"})
freqai_conf.get("freqai", {}).get("feature_parameters", {}).update( freqai_conf.get("freqai", {}).get("feature_parameters", {}).update(