From fda3a2827bbebc2a34d97620e41399e57d0a866b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 14 Oct 2022 16:20:49 +0000 Subject: [PATCH 1/6] add list-freqAI models command --- freqtrade/commands/__init__.py | 6 +++--- freqtrade/commands/arguments.py | 24 ++++++++++++++++++------ freqtrade/commands/list_commands.py | 15 +++++++++++++++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index d93ed1e09..788657cc8 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -15,9 +15,9 @@ from freqtrade.commands.db_commands import start_convert_db from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, start_new_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show -from freqtrade.commands.list_commands import (start_list_exchanges, start_list_markets, - start_list_strategies, start_list_timeframes, - start_show_trades) +from freqtrade.commands.list_commands import (start_list_exchanges, start_list_freqAI_models, + start_list_markets, start_list_strategies, + start_list_timeframes, start_show_trades) from freqtrade.commands.optimize_commands import (start_backtesting, start_backtesting_show, start_edge, start_hyperopt) from freqtrade.commands.pairlist_commands import start_test_pairlist diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 97d8cc130..57689db0a 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -41,6 +41,8 @@ ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column", "print_colorized", "recursive_strategy_search"] +ARGS_LIST_FREQAIMODELS = ["freqaimodel_path", "print_one_column", "print_colorized"] + ARGS_LIST_HYPEROPTS = ["hyperopt_path", "print_one_column", "print_colorized"] ARGS_BACKTEST_SHOW = ["exportfilename", "backtest_show_pair_list"] @@ -106,8 +108,8 @@ ARGS_ANALYZE_ENTRIES_EXITS = ["exportfilename", "analysis_groups", "enter_reason "exit_reason_list", "indicator_list"] NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", - "list-markets", "list-pairs", "list-strategies", "list-data", - "hyperopt-list", "hyperopt-show", "backtest-filter", + "list-markets", "list-pairs", "list-strategies", "list-freqaimodels", + "list-data", "hyperopt-list", "hyperopt-show", "backtest-filter", "plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] @@ -192,10 +194,11 @@ class Arguments: start_create_userdir, start_download_data, start_edge, start_hyperopt, start_hyperopt_list, start_hyperopt_show, start_install_ui, start_list_data, start_list_exchanges, - start_list_markets, start_list_strategies, - start_list_timeframes, start_new_config, start_new_strategy, - start_plot_dataframe, start_plot_profit, start_show_trades, - start_test_pairlist, start_trading, start_webserver) + start_list_freqAI_models, start_list_markets, + start_list_strategies, start_list_timeframes, + start_new_config, start_new_strategy, start_plot_dataframe, + start_plot_profit, start_show_trades, start_test_pairlist, + start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -362,6 +365,15 @@ class Arguments: list_strategies_cmd.set_defaults(func=start_list_strategies) self._build_args(optionlist=ARGS_LIST_STRATEGIES, parser=list_strategies_cmd) + # Add list-freqAI Models subcommand + list_freqaimodels_cmd = subparsers.add_parser( + 'list-freqaimodels', + help='Print available freqAI models.', + parents=[_common_parser], + ) + list_freqaimodels_cmd.set_defaults(func=start_list_freqAI_models) + self._build_args(optionlist=ARGS_LIST_FREQAIMODELS, parser=list_freqaimodels_cmd) + # Add list-timeframes subcommand list_timeframes_cmd = subparsers.add_parser( 'list-timeframes', diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 7ab9202aa..4e0623081 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -90,6 +90,21 @@ def start_list_strategies(args: Dict[str, Any]) -> None: _print_objs_tabular(strategy_objs, config.get('print_colorized', False)) +def start_list_freqAI_models(args: Dict[str, Any]) -> None: + """ + Print files with FreqAI models custom classes available in the directory + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver + model_objs = FreqaiModelResolver.search_all_objects(config, not args['print_one_column']) + # Sort alphabetically + model_objs = sorted(model_objs, key=lambda x: x['name']) + if args['print_one_column']: + print('\n'.join([s['name'] for s in model_objs])) + else: + _print_objs_tabular(model_objs, config.get('print_colorized', False)) + + def start_list_timeframes(args: Dict[str, Any]) -> None: """ Print timeframes available on Exchange From 2ef315e8c2fbd6f8e4b38b28893fa8f4aef275ea Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 14 Oct 2022 18:24:15 +0200 Subject: [PATCH 2/6] Add documentation for list-freqaimodels --- docs/utils.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/utils.md b/docs/utils.md index 174fa0527..ee8793159 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -169,6 +169,43 @@ Example: Search dedicated strategy path. freqtrade list-strategies --strategy-path ~/.freqtrade/strategies/ ``` +## List freqAI models + +Use the `list-freqaimodels` subcommand to see all freqAI models available. + +This subcommand is useful for finding problems in your environment with loading freqAI models: modules with models that contain errors and failed to load are printed in red (LOAD FAILED), while models with duplicate names are printed in yellow (DUPLICATE NAME). + +``` +usage: freqtrade list-freqaimodels [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [--freqaimodel-path PATH] [-1] [--no-color] + +optional arguments: + -h, --help show this help message and exit + --freqaimodel-path PATH + Specify additional lookup path for freqaimodels. + -1, --one-column Print output in one column. + --no-color Disable colorization of hyperopt results. May be + useful if you are redirecting output to a file. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH, --data-dir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +``` + ## List Exchanges Use the `list-exchanges` subcommand to see the exchanges available for the bot. From 4bfe58706be2acb9d77384cc52a98fceba3bc820 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 14 Oct 2022 19:49:06 +0200 Subject: [PATCH 3/6] Generalize "path" variables for resolvers --- freqtrade/resolvers/freqaimodel_resolver.py | 2 +- freqtrade/resolvers/iresolver.py | 5 +++++ freqtrade/resolvers/strategy_resolver.py | 9 +-------- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/freqtrade/resolvers/freqaimodel_resolver.py b/freqtrade/resolvers/freqaimodel_resolver.py index aa5228ca1..48c3facac 100644 --- a/freqtrade/resolvers/freqaimodel_resolver.py +++ b/freqtrade/resolvers/freqaimodel_resolver.py @@ -26,6 +26,7 @@ class FreqaiModelResolver(IResolver): initial_search_path = ( Path(__file__).parent.parent.joinpath("freqai/prediction_models").resolve() ) + extra_path = "freqaimodel_path" @staticmethod def load_freqaimodel(config: Config) -> IFreqaiModel: @@ -50,7 +51,6 @@ class FreqaiModelResolver(IResolver): freqaimodel_name, config, kwargs={"config": config}, - extra_dir=config.get("freqaimodel_path"), ) return freqaimodel diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 5c4ba1568..0b484394a 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -42,6 +42,8 @@ class IResolver: object_type_str: str user_subdir: Optional[str] = None initial_search_path: Optional[Path] + # Optional config setting containing a path (strategy_path, freqaimodel_path) + extra_path: Optional[str] = None @classmethod def build_search_paths(cls, config: Config, user_subdir: Optional[str] = None, @@ -58,6 +60,9 @@ class IResolver: for dir in extra_dirs: abs_paths.insert(0, Path(dir).resolve()) + if cls.extra_path and (extra := config.get(cls.extra_path)): + abs_paths.insert(0, Path(extra).resolve()) + return abs_paths @classmethod diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index ae27df3c5..67df49dcb 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -30,6 +30,7 @@ class StrategyResolver(IResolver): object_type_str = "Strategy" user_subdir = USERPATH_STRATEGIES initial_search_path = None + extra_path = "strategy_path" @staticmethod def load_strategy(config: Config = None) -> IStrategy: @@ -268,14 +269,6 @@ class StrategyResolver(IResolver): "or contains Python code errors." ) - @classmethod - def build_search_paths(cls, config: Config, user_subdir: Optional[str] = None, - extra_dirs: List[str] = []) -> List[Path]: - - if 'strategy_path' in config and config['strategy_path'] not in extra_dirs: - extra_dirs = [config['strategy_path']] + extra_dirs - return super().build_search_paths(config, user_subdir, extra_dirs) - def warn_deprecated_setting(strategy: IStrategy, old: str, new: str, error=False): if hasattr(strategy, old): From 93ad3810fd12f13457f3c9eb8e48ae50eed71d3d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 15 Oct 2022 08:20:06 +0200 Subject: [PATCH 4/6] Add test for list-freqaimodels --- tests/commands/test_commands.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 28515a28a..d3bceb004 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -18,6 +18,7 @@ from freqtrade.commands import (start_backtesting_show, start_convert_data, star from freqtrade.commands.db_commands import start_convert_db from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) +from freqtrade.commands.list_commands import start_list_freqAI_models from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException @@ -944,6 +945,34 @@ def test_start_list_strategies(capsys): assert str(Path("broken_strats/broken_futures_strategies.py")) in captured.out +def test_start_list_freqAI_models(capsys): + + args = [ + "list-freqaimodels", + "-1" + ] + pargs = get_args(args) + pargs['config'] = None + start_list_freqAI_models(pargs) + captured = capsys.readouterr() + assert "LightGBMClassifier" in captured.out + assert "LightGBMRegressor" in captured.out + assert "XGBoostRegressor" in captured.out + assert "/LightGBMRegressor.py" not in captured.out + + args = [ + "list-freqaimodels", + ] + pargs = get_args(args) + pargs['config'] = None + start_list_freqAI_models(pargs) + captured = capsys.readouterr() + assert "LightGBMClassifier" in captured.out + assert "LightGBMRegressor" in captured.out + assert "XGBoostRegressor" in captured.out + assert "/LightGBMRegressor.py" in captured.out + + def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): patch_exchange(mocker, mock_markets=True) mocker.patch.multiple('freqtrade.exchange.Exchange', From 6cb14148aab461a8ca6cf24af3a11227c16b5114 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 16 Oct 2022 10:36:58 +0200 Subject: [PATCH 5/6] Fix random test failure due to catboost bug https://github.com/catboost/catboost/issues/2195 --- freqtrade/freqai/prediction_models/CatboostClassifier.py | 4 +++- freqtrade/freqai/prediction_models/CatboostRegressor.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/prediction_models/CatboostClassifier.py b/freqtrade/freqai/prediction_models/CatboostClassifier.py index 063c76d1b..ca1d8ece0 100644 --- a/freqtrade/freqai/prediction_models/CatboostClassifier.py +++ b/freqtrade/freqai/prediction_models/CatboostClassifier.py @@ -1,4 +1,5 @@ import logging +import sys from pathlib import Path from typing import Any, Dict @@ -48,6 +49,7 @@ class CatboostClassifier(BaseClassifierModel): init_model = self.get_init_model(dk.pair) - cbr.fit(X=train_data, eval_set=test_data, init_model=init_model) + cbr.fit(X=train_data, eval_set=test_data, init_model=init_model, + log_cout=sys.stdout, log_cerr=sys.stderr) return cbr diff --git a/freqtrade/freqai/prediction_models/CatboostRegressor.py b/freqtrade/freqai/prediction_models/CatboostRegressor.py index 2978f6679..4b17a703b 100644 --- a/freqtrade/freqai/prediction_models/CatboostRegressor.py +++ b/freqtrade/freqai/prediction_models/CatboostRegressor.py @@ -1,4 +1,5 @@ import logging +import sys from pathlib import Path from typing import Any, Dict @@ -47,6 +48,7 @@ class CatboostRegressor(BaseRegressionModel): **self.model_training_parameters, ) - model.fit(X=train_data, eval_set=test_data, init_model=init_model) + model.fit(X=train_data, eval_set=test_data, init_model=init_model, + log_cout=sys.stdout, log_cerr=sys.stderr) return model From 441032be25dcabff2fc1bc8d0b7e7f855f7a0da8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 17 Oct 2022 19:48:27 +0200 Subject: [PATCH 6/6] Fix sys.stdout bug for CatboostRegressorMultiTarget --- .../prediction_models/CatboostRegressorMultiTarget.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/prediction_models/CatboostRegressorMultiTarget.py b/freqtrade/freqai/prediction_models/CatboostRegressorMultiTarget.py index de7a73e3a..976d0b29b 100644 --- a/freqtrade/freqai/prediction_models/CatboostRegressorMultiTarget.py +++ b/freqtrade/freqai/prediction_models/CatboostRegressorMultiTarget.py @@ -1,4 +1,5 @@ import logging +import sys from pathlib import Path from typing import Any, Dict @@ -58,8 +59,10 @@ class CatboostRegressorMultiTarget(BaseRegressionModel): fit_params = [] for i in range(len(eval_sets)): - fit_params.append( - {'eval_set': eval_sets[i], 'init_model': init_models[i]}) + fit_params.append({ + 'eval_set': eval_sets[i], 'init_model': init_models[i], + 'log_cout': sys.stdout, 'log_cerr': sys.stderr, + }) model = FreqaiMultiOutputRegressor(estimator=cbr) thread_training = self.freqai_info.get('multitarget_parallel_training', False)