From 49bfa556bfa413833fbb59b6bde2e3de6de9f65f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 Oct 2022 21:04:40 +0200 Subject: [PATCH 1/7] Update CI to test against python 3.11 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17c0efd6d..191a10d1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: os: [ ubuntu-20.04, ubuntu-22.04 ] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 @@ -115,7 +115,7 @@ jobs: strategy: matrix: os: [ macos-latest ] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 @@ -212,7 +212,7 @@ jobs: strategy: matrix: os: [ windows-latest ] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 From 684d310ea0d7c05996fe08b6e509cc1d6771c61e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Feb 2023 13:20:26 +0100 Subject: [PATCH 2/7] Limit catboost to python <3.11 --- requirements-freqai.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 5b27ecf95..bc0be85e5 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -5,7 +5,7 @@ # Required for freqai scikit-learn==1.1.3 joblib==1.2.0 -catboost==1.1.1; platform_machine != 'aarch64' +catboost==1.1.1; platform_machine != 'aarch64' and python_version < '3.11' lightgbm==3.3.5 xgboost==1.7.4 tensorboard==2.12.0 From 7a7f16b6589fc2b36ceb0bd17651405bb8f5a4ce Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Feb 2023 13:27:28 +0100 Subject: [PATCH 3/7] Skip catboost tests on py3.11 --- tests/freqai/test_freqai_interface.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index cdfc943af..7d229434f 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -1,5 +1,6 @@ import platform import shutil +import sys from pathlib import Path from unittest.mock import MagicMock @@ -17,6 +18,10 @@ from tests.conftest import EXMS, create_mock_trades, get_patched_exchange, log_h from tests.freqai.conftest import get_patched_freqai_strategy, make_rl_config +def is_py11() -> bool: + return sys.version_info >= (3, 11) + + def is_arm() -> bool: machine = platform.machine() return "arm" in machine or "aarch64" in machine @@ -41,7 +46,7 @@ def is_mac() -> bool: def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, dbscan, float32, can_short, shuffle, buffer): - if is_arm() and model == 'CatboostRegressor': + if (is_arm() or is_py11()) and model == 'CatboostRegressor': pytest.skip("CatBoost is not supported on ARM") if is_mac() and not is_arm() and 'Reinforcement' in model: @@ -117,7 +122,7 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, ('CatboostClassifierMultiTarget', "freqai_test_multimodel_classifier_strat") ]) def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model, strat): - if is_arm() and 'Catboost' in model: + if (is_arm() or is_py11()) and 'Catboost' in model: pytest.skip("CatBoost is not supported on ARM") freqai_conf.update({"timerange": "20180110-20180130"}) @@ -159,7 +164,7 @@ def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model, s 'XGBoostRFClassifier', ]) def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): - if is_arm() and model == 'CatboostClassifier': + if (is_arm() or is_py11()) and model == 'CatboostClassifier': pytest.skip("CatBoost is not supported on ARM") freqai_conf.update({"freqaimodel": model}) @@ -208,7 +213,7 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog): freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) freqai_conf['runmode'] = RunMode.BACKTEST - if is_arm() and "Catboost" in model: + if (is_arm() or is_py11()) and "Catboost" in model: pytest.skip("CatBoost is not supported on ARM") if is_mac() and 'Reinforcement' in model: From b1a5776f1480461f87403e136df4eb591a3b0c97 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Feb 2023 15:21:12 +0100 Subject: [PATCH 4/7] Skip reinforcement learning for python 3.11 --- requirements-freqai-rl.txt | 8 ++++---- tests/freqai/test_freqai_interface.py | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index c242af43e..4de7d8fab 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -2,9 +2,9 @@ -r requirements-freqai.txt # Required for freqai-rl -torch==1.13.1 -stable-baselines3==1.7.0 -sb3-contrib==1.7.0 +torch==1.13.1; python_version < '3.11' +stable-baselines3==1.7.0; python_version < '3.11' +sb3-contrib==1.7.0; python_version < '3.11' # Gym is forced to this version by stable-baselines3. setuptools==65.5.1 # Should be removed when gym is fixed. -gym==0.21 +gym==0.21; python_version < '3.11' diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 7d229434f..0584ed9f5 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -51,7 +51,8 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, if is_mac() and not is_arm() and 'Reinforcement' in model: pytest.skip("Reinforcement learning module not available on intel based Mac OS") - + if is_py11() and 'Reinforcement' in model: + pytest.skip("Reinforcement learning currently not available on python 3.11.") model_save_ext = 'joblib' freqai_conf.update({"freqaimodel": model}) freqai_conf.update({"timerange": "20180110-20180130"}) @@ -218,6 +219,8 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) if is_mac() and 'Reinforcement' in model: pytest.skip("Reinforcement learning module not available on intel based Mac OS") + if is_py11() and 'Reinforcement' in model: + pytest.skip("Reinforcement learning currently not available on python 3.11.") Trade.use_db = False freqai_conf.update({"freqaimodel": model}) @@ -514,6 +517,8 @@ def test_get_state_info(mocker, freqai_conf, dp_exists, caplog, tickers): if is_mac(): pytest.skip("Reinforcement learning module not available on intel based Mac OS") + if is_py11(): + pytest.skip("Reinforcement learning currently not available on python 3.11.") freqai_conf.update({"freqaimodel": "ReinforcementLearner"}) freqai_conf.update({"timerange": "20180110-20180130"}) From 38050b5346fa563e9592bdb3258602adb1b753c1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Feb 2023 15:25:47 +0100 Subject: [PATCH 5/7] Simplify "model-run" conditions --- tests/freqai/test_freqai_interface.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 0584ed9f5..f8bee3659 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -32,6 +32,17 @@ def is_mac() -> bool: return "Darwin" in machine +def can_run_model(model: str) -> None: + if (is_arm() or is_py11()) and "Catboost" in model: + pytest.skip("CatBoost is not supported on ARM") + + if is_mac() and not is_arm() and 'Reinforcement' in model: + pytest.skip("Reinforcement learning module not available on intel based Mac OS") + + if is_py11() and 'Reinforcement' in model: + pytest.skip("Reinforcement learning currently not available on python 3.11.") + + @pytest.mark.parametrize('model, pca, dbscan, float32, can_short, shuffle, buffer', [ ('LightGBMRegressor', True, False, True, True, False, 0), ('XGBoostRegressor', False, True, False, True, False, 10), @@ -46,13 +57,7 @@ def is_mac() -> bool: def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, dbscan, float32, can_short, shuffle, buffer): - if (is_arm() or is_py11()) and model == 'CatboostRegressor': - pytest.skip("CatBoost is not supported on ARM") - - if is_mac() and not is_arm() and 'Reinforcement' in model: - pytest.skip("Reinforcement learning module not available on intel based Mac OS") - if is_py11() and 'Reinforcement' in model: - pytest.skip("Reinforcement learning currently not available on python 3.11.") + can_run_model(model) model_save_ext = 'joblib' freqai_conf.update({"freqaimodel": model}) freqai_conf.update({"timerange": "20180110-20180130"}) @@ -212,15 +217,11 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): ], ) def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog): + can_run_model(model) + freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) freqai_conf['runmode'] = RunMode.BACKTEST - if (is_arm() or is_py11()) and "Catboost" in model: - pytest.skip("CatBoost is not supported on ARM") - if is_mac() and 'Reinforcement' in model: - pytest.skip("Reinforcement learning module not available on intel based Mac OS") - if is_py11() and 'Reinforcement' in model: - pytest.skip("Reinforcement learning currently not available on python 3.11.") Trade.use_db = False freqai_conf.update({"freqaimodel": model}) From 5d0e14b5644d100ed2a5fe7ddcc6b987cbc69741 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Mar 2023 07:07:09 +0100 Subject: [PATCH 6/7] Don't mock full modules --- tests/rpc/test_rpc_apiserver.py | 4 ++-- tests/rpc/test_rpc_telegram.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index b104ec854..f898dd476 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1179,7 +1179,7 @@ def test_api_force_entry(botclient, mocker, fee, endpoint): ftbot.config['force_entry_enable'] = True fbuy_mock = MagicMock(return_value=None) - mocker.patch("freqtrade.rpc.RPC._rpc_force_entry", fbuy_mock) + mocker.patch("freqtrade.rpc.rpc.RPC._rpc_force_entry", fbuy_mock) rc = client_post(client, f"{BASE_URI}/{endpoint}", data={"pair": "ETH/BTC"}) assert_response(rc) @@ -1205,7 +1205,7 @@ def test_api_force_entry(botclient, mocker, fee, endpoint): strategy=CURRENT_TEST_STRATEGY, trading_mode=TradingMode.SPOT )) - mocker.patch("freqtrade.rpc.RPC._rpc_force_entry", fbuy_mock) + mocker.patch("freqtrade.rpc.rpc.RPC._rpc_force_entry", fbuy_mock) rc = client_post(client, f"{BASE_URI}/{endpoint}", data={"pair": "ETH/BTC"}) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 3e1421cb5..7b3411113 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1209,7 +1209,7 @@ def test_force_enter_handle(default_conf, update, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) fbuy_mock = MagicMock(return_value=None) - mocker.patch('freqtrade.rpc.RPC._rpc_force_entry', fbuy_mock) + mocker.patch('freqtrade.rpc.rpc.RPC._rpc_force_entry', fbuy_mock) telegram, freqtradebot, _ = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot) @@ -1226,7 +1226,7 @@ def test_force_enter_handle(default_conf, update, mocker) -> None: # Reset and retry with specified price fbuy_mock = MagicMock(return_value=None) - mocker.patch('freqtrade.rpc.RPC._rpc_force_entry', fbuy_mock) + mocker.patch('freqtrade.rpc.rpc.RPC._rpc_force_entry', fbuy_mock) # /forcelong ETH/BTC 0.055 context = MagicMock() context.args = ["ETH/BTC", "0.055"] @@ -1255,7 +1255,7 @@ def test_force_enter_no_pair(default_conf, update, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) fbuy_mock = MagicMock(return_value=None) - mocker.patch('freqtrade.rpc.RPC._rpc_force_entry', fbuy_mock) + mocker.patch('freqtrade.rpc.rpc.RPC._rpc_force_entry', fbuy_mock) telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) From 5b0c1437130a502f322e915a4095b43e68ba88b2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Mar 2023 19:29:41 +0100 Subject: [PATCH 7/7] Update some comments about 3.11 --- docs/freqai.md | 4 ++++ docs/windows_installation.md | 2 +- setup.cfg | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/freqai.md b/docs/freqai.md index 5c6b5b2ce..ef8efb840 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -71,6 +71,10 @@ pip install -r requirements-freqai.txt !!! Note Catboost will not be installed on arm devices (raspberry, Mac M1, ARM based VPS, ...), since it does not provide wheels for this platform. +!!! Note "python 3.11" + Some dependencies (Catboost, Torch) currently don't support python 3.11. Freqtrade therefore only supports python 3.10 for these models/dependencies. + Tests involving these dependencies are skipped on 3.11. + ### Usage with docker If you are using docker, a dedicated tag with FreqAI dependencies is available as `:freqai`. As such - you can replace the image line in your docker compose file with `image: freqtradeorg/freqtrade:develop_freqai`. This image contains the regular FreqAI dependencies. Similar to native installs, Catboost will not be available on ARM based devices. diff --git a/docs/windows_installation.md b/docs/windows_installation.md index 1b0d9d724..43d6728ee 100644 --- a/docs/windows_installation.md +++ b/docs/windows_installation.md @@ -26,7 +26,7 @@ Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7 As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial pre-compiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which need to be downloaded and installed using `pip install TA_Lib-0.4.25-cp38-cp38-win_amd64.whl` (make sure to use the version matching your python version). -Freqtrade provides these dependencies for the latest 3 Python versions (3.8, 3.9 and 3.10) and for 64bit Windows. +Freqtrade provides these dependencies for the latest 3 Python versions (3.8, 3.9, 3.10 and 3.11) and for 64bit Windows. Other versions must be downloaded from the above link. ``` powershell diff --git a/setup.cfg b/setup.cfg index 60ec8a75f..b54b62619 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Operating System :: MacOS Operating System :: Unix Topic :: Office/Business :: Financial :: Investment