From 27cc73f47ed2936f3f3e2a98eee3095af302b3f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Sep 2019 14:39:28 +0200 Subject: [PATCH 1/6] Dynamically import hyperopt modules --- freqtrade/optimize/__init__.py | 12 +++++++----- tests/optimize/test_hyperopt.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 7a3c290bf..3adf5eb43 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,9 +1,7 @@ import logging from typing import Any, Dict -from filelock import FileLock, Timeout - -from freqtrade import DependencyException, constants +from freqtrade import DependencyException, constants, OperationalException from freqtrade.state import RunMode from freqtrade.utils import setup_utils_configuration @@ -53,8 +51,12 @@ def start_hyperopt(args: Dict[str, Any]) -> None: :return: None """ # Import here to avoid loading hyperopt module when it's not used - from freqtrade.optimize.hyperopt import Hyperopt - + try: + from filelock import FileLock, Timeout + from freqtrade.optimize.hyperopt import Hyperopt + except ImportError as e: + raise OperationalException( + f"{e}. Please ensure that the hyperopt dependencies are installed.") from e # Initialize configuration config = setup_configuration(args, RunMode.HYPEROPT) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 0888c79d4..8ceea6caa 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -190,6 +190,37 @@ def test_hyperoptlossresolver_wrongname(mocker, default_conf, caplog) -> None: HyperOptLossResolver(default_conf, ).hyperopt +def test_start_not_installed(mocker, default_conf, caplog) -> None: + start_mock = MagicMock() + patched_configuration_load_config_file(mocker, default_conf) + # Source of this test-method: https://stackoverflow.com/questions/2481511/mocking-importerror-in-python + import builtins + realimport = builtins.__import__ + + def mockedimport(name, *args, **kwargs): + if name == "filelock": + raise ImportError("No module named 'filelock'") + return realimport(name, *args, **kwargs) + + builtins.__import__ = mockedimport + + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) + patch_exchange(mocker) + + args = [ + '--config', 'config.json', + 'hyperopt', + '--epochs', '5' + ] + args = get_args(args) + + with pytest.raises(OperationalException, match=r"Please ensure that the hyperopt dependencies"): + start_hyperopt(args) + + # restore previous importfunction + builtins.__import__ = realimport + + def test_start(mocker, default_conf, caplog) -> None: start_mock = MagicMock() patched_configuration_load_config_file(mocker, default_conf) From 47b6b56566bd90752b44df044eaa66c5ca9c601f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Sep 2019 14:53:36 +0200 Subject: [PATCH 2/6] Reorg dependencies to have hyperopt seperated --- environment.yml | 11 ++++++----- requirements-common.txt | 8 +------- requirements-dev.txt | 1 + requirements-hyperopt.txt | 9 +++++++++ requirements.txt | 1 - 5 files changed, 17 insertions(+), 13 deletions(-) create mode 100644 requirements-hyperopt.txt diff --git a/environment.yml b/environment.yml index cd3350fd5..4e8c1efcc 100644 --- a/environment.yml +++ b/environment.yml @@ -9,25 +9,26 @@ dependencies: - wheel - numpy - pandas - - scipy - SQLAlchemy - - scikit-learn - arrow - requests - urllib3 - wrapt - - joblib - jsonschema - tabulate - python-rapidjson - - filelock - flask - python-dotenv - cachetools - - scikit-optimize - python-telegram-bot # Optional for plotting - plotly + # Optional for hyperopt + - scipy + - scikit-optimize + - scikit-learn + - filelock + - joblib # Optional for development - flake8 - pytest diff --git a/requirements-common.txt b/requirements-common.txt index 70a37e695..f10134203 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -8,21 +8,15 @@ cachetools==3.1.1 requests==2.22.0 urllib3==1.25.5 wrapt==1.11.2 -scikit-learn==0.21.3 -joblib==0.13.2 jsonschema==3.0.2 TA-Lib==0.4.17 tabulate==0.8.3 coinmarketcap==5.0.3 -# Required for hyperopt -scikit-optimize==0.5.2 -filelock==3.0.12 - # find first, C search in arrays py_find_1st==1.1.4 -#Load ticker files 30% faster +# Load ticker files 30% faster python-rapidjson==0.8.0 # Notify systemd diff --git a/requirements-dev.txt b/requirements-dev.txt index 2678130f3..dcf2c7217 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ # Include all requirements to run the bot. -r requirements.txt -r requirements-plot.txt +-r requirements-hyperopt.txt coveralls==1.8.2 flake8==3.7.8 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt new file mode 100644 index 000000000..bb0ad60f0 --- /dev/null +++ b/requirements-hyperopt.txt @@ -0,0 +1,9 @@ +# Include all requirements to run the bot. +# -r requirements.txt + +# Required for hyperopt +scipy==1.3.1 +scikit-learn==0.21.3 +scikit-optimize==0.5.2 +filelock==3.0.12 +joblib==0.13.2 diff --git a/requirements.txt b/requirements.txt index 9a723fee4..2767180ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ numpy==1.17.2 pandas==0.25.1 -scipy==1.3.1 From d2f247307092e5df9c67107f491655e991163d04 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Sep 2019 15:03:40 +0200 Subject: [PATCH 3/6] install hyperopt seperately ([hyperopt]) --- setup.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index ca94dd338..2eb09e589 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,13 @@ if readme_file.is_file(): # Requirements used for submodules api = ['flask'] plot = ['plotly>=4.0'] +hyperopt = [ + 'scipy', + 'scikit-learn', + 'scikit-optimize', + 'filelock', + 'joblib', + ] develop = [ 'coveralls', @@ -38,7 +45,7 @@ jupyter = [ 'ipykernel', ] -all_extra = api + plot + develop + jupyter +all_extra = api + plot + develop + jupyter + hyperopt setup(name='freqtrade', version=__version__, @@ -62,14 +69,10 @@ setup(name='freqtrade', 'requests', 'urllib3', 'wrapt', - 'scikit-learn', - 'joblib', 'jsonschema', 'TA-Lib', 'tabulate', 'coinmarketcap', - 'scikit-optimize', - 'filelock', 'py_find_1st', 'python-rapidjson', 'sdnotify', @@ -77,15 +80,14 @@ setup(name='freqtrade', # from requirements.txt 'numpy', 'pandas', - 'scipy', ], extras_require={ 'api': api, 'dev': all_extra, 'plot': plot, - 'all': all_extra, 'jupyter': jupyter, - + 'hyperopt': hyperopt, + 'all': all_extra, }, include_package_data=True, zip_safe=False, From d05db077e29666e4539bd929d67edb252be3b2dc Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Sep 2019 08:56:06 +0200 Subject: [PATCH 4/6] Update PI install documentation and dockerfile --- Dockerfile.pi | 4 ++-- docs/installation.md | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Dockerfile.pi b/Dockerfile.pi index 1b9c4c579..85ba5892f 100644 --- a/Dockerfile.pi +++ b/Dockerfile.pi @@ -22,13 +22,13 @@ RUN tar -xzf /freqtrade/ta-lib-0.4.0-src.tar.gz \ ENV LD_LIBRARY_PATH /usr/local/lib # Install berryconda -RUN wget https://github.com/jjhelmus/berryconda/releases/download/v2.0.0/Berryconda3-2.0.0-Linux-armv7l.sh \ +RUN wget -q https://github.com/jjhelmus/berryconda/releases/download/v2.0.0/Berryconda3-2.0.0-Linux-armv7l.sh \ && bash ./Berryconda3-2.0.0-Linux-armv7l.sh -b \ && rm Berryconda3-2.0.0-Linux-armv7l.sh # Install dependencies COPY requirements-common.txt /freqtrade/ -RUN ~/berryconda3/bin/conda install -y numpy pandas scipy \ +RUN ~/berryconda3/bin/conda install -y numpy pandas \ && ~/berryconda3/bin/pip install -r requirements-common.txt --no-cache-dir # Install and execute diff --git a/docs/installation.md b/docs/installation.md index f15cc356c..641879ff1 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -99,8 +99,8 @@ sudo apt-get install build-essential git Before installing FreqTrade on a Raspberry Pi running the official Raspbian Image, make sure you have at least Python 3.6 installed. The default image only provides Python 3.5. Probably the easiest way to get a recent version of python is [miniconda](https://repo.continuum.io/miniconda/). -The following assumes that miniconda3 is installed and available in your environment. Last miniconda3 installation file use python 3.4, we will update to python 3.6 on this installation. -It's recommended to use (mini)conda for this as installation/compilation of `numpy`, `scipy` and `pandas` takes a long time. +The following assumes that miniconda3 is installed and available in your environment. Since the last miniconda3 installation file uses python 3.4, we will update to python 3.6 on this installation. +It's recommended to use (mini)conda for this as installation/compilation of `numpy` and `pandas` takes a long time. Additional package to install on your Raspbian, `libffi-dev` required by cryptography (from python-telegram-bot). @@ -109,13 +109,17 @@ conda config --add channels rpi conda install python=3.6 conda create -n freqtrade python=3.6 conda activate freqtrade -conda install scipy pandas numpy +conda install pandas numpy sudo apt install libffi-dev python3 -m pip install -r requirements-common.txt python3 -m pip install -e . ``` +!!! Note + This does not install hyperopt dependencies. To install these, please use `python3 -m pip install -e .[hyperopt]`. + We do not advise to run hyperopt on a Raspberry Pi, since this is a very resource-heavy operation, which should be done on powerfull machine. + ### Common #### 1. Install TA-Lib @@ -175,7 +179,6 @@ cp config.json.example config.json ``` bash python3 -m pip install --upgrade pip -python3 -m pip install -r requirements.txt python3 -m pip install -e . ``` From e9de088209736e1f1f255c4339575ff7b34c38f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Sep 2019 11:54:57 +0200 Subject: [PATCH 5/6] Add import-fails code as a fixture --- tests/conftest.py | 21 +++++++++++++++++++++ tests/optimize/test_hyperopt.py | 15 +-------------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8a5ba6683..6a0a74b5b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1054,3 +1054,24 @@ def rpc_balance(): def testdatadir() -> Path: """Return the path where testdata files are stored""" return (Path(__file__).parent / "testdata").resolve() + + +@pytest.fixture(scope="function") +def import_fails() -> None: + # Source of this test-method: + # https://stackoverflow.com/questions/2481511/mocking-importerror-in-python + import builtins + realimport = builtins.__import__ + + def mockedimport(name, *args, **kwargs): + if name in ["filelock"]: + raise ImportError(f"No module named '{name}'") + return realimport(name, *args, **kwargs) + + builtins.__import__ = mockedimport + + # Run test - then cleanup + yield + + # restore previous importfunction + builtins.__import__ = realimport diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 8ceea6caa..55f94f572 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -190,19 +190,9 @@ def test_hyperoptlossresolver_wrongname(mocker, default_conf, caplog) -> None: HyperOptLossResolver(default_conf, ).hyperopt -def test_start_not_installed(mocker, default_conf, caplog) -> None: +def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None: start_mock = MagicMock() patched_configuration_load_config_file(mocker, default_conf) - # Source of this test-method: https://stackoverflow.com/questions/2481511/mocking-importerror-in-python - import builtins - realimport = builtins.__import__ - - def mockedimport(name, *args, **kwargs): - if name == "filelock": - raise ImportError("No module named 'filelock'") - return realimport(name, *args, **kwargs) - - builtins.__import__ = mockedimport mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) patch_exchange(mocker) @@ -217,9 +207,6 @@ def test_start_not_installed(mocker, default_conf, caplog) -> None: with pytest.raises(OperationalException, match=r"Please ensure that the hyperopt dependencies"): start_hyperopt(args) - # restore previous importfunction - builtins.__import__ = realimport - def test_start(mocker, default_conf, caplog) -> None: start_mock = MagicMock() From eb07f1fee9b5763a0e635c586b55e5d4d5243325 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 Sep 2019 09:31:31 +0200 Subject: [PATCH 6/6] Fix typo --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 641879ff1..081d7e0cf 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -118,7 +118,7 @@ python3 -m pip install -e . !!! Note This does not install hyperopt dependencies. To install these, please use `python3 -m pip install -e .[hyperopt]`. - We do not advise to run hyperopt on a Raspberry Pi, since this is a very resource-heavy operation, which should be done on powerfull machine. + We do not advise to run hyperopt on a Raspberry Pi, since this is a very resource-heavy operation, which should be done on powerful machine. ### Common