diff --git a/Dockerfile.pi b/Dockerfile.pi index 1b9c4c579..85ba5892f 100644 --- a/Dockerfile.pi +++ b/Dockerfile.pi @@ -22,13 +22,13 @@ RUN tar -xzf /freqtrade/ta-lib-0.4.0-src.tar.gz \ ENV LD_LIBRARY_PATH /usr/local/lib # Install berryconda -RUN wget https://github.com/jjhelmus/berryconda/releases/download/v2.0.0/Berryconda3-2.0.0-Linux-armv7l.sh \ +RUN wget -q https://github.com/jjhelmus/berryconda/releases/download/v2.0.0/Berryconda3-2.0.0-Linux-armv7l.sh \ && bash ./Berryconda3-2.0.0-Linux-armv7l.sh -b \ && rm Berryconda3-2.0.0-Linux-armv7l.sh # Install dependencies COPY requirements-common.txt /freqtrade/ -RUN ~/berryconda3/bin/conda install -y numpy pandas scipy \ +RUN ~/berryconda3/bin/conda install -y numpy pandas \ && ~/berryconda3/bin/pip install -r requirements-common.txt --no-cache-dir # Install and execute diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 0b0561d3d..f44400e32 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -184,10 +184,6 @@ optional arguments: Specify max_open_trades to use. --stake_amount STAKE_AMOUNT Specify stake_amount. - -r, --refresh-pairs-cached - Refresh the pairs files in tests/testdata with the - latest data from the exchange. Use it if you want to - run your optimization commands with up-to-date data. --eps, --enable-position-stacking Allow buying the same pair multiple times (position stacking). @@ -245,10 +241,6 @@ optional arguments: Specify max_open_trades to use. --stake_amount STAKE_AMOUNT Specify stake_amount. - -r, --refresh-pairs-cached - Refresh the pairs files in tests/testdata with the - latest data from the exchange. Use it if you want to - run your optimization commands with up-to-date data. --customhyperopt NAME Specify hyperopt class name (default: `DefaultHyperOpts`). @@ -310,10 +302,6 @@ optional arguments: Specify max_open_trades to use. --stake_amount STAKE_AMOUNT Specify stake_amount. - -r, --refresh-pairs-cached - Refresh the pairs files in tests/testdata with the - latest data from the exchange. Use it if you want to - run your optimization commands with up-to-date data. --stoplosses STOPLOSS_RANGE Defines a range of stoploss against which edge will assess the strategy the format is "min,max,step" diff --git a/docs/configuration.md b/docs/configuration.md index a1c0d1a75..0d902766a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,14 +1,15 @@ # Configure the bot -This page explains how to configure the bot. +Freqtrade has many configurable features and possibilities. +By default, these settings are configured via the configuration file (see below). ## The Freqtrade configuration file The bot uses a set of configuration parameters during its operation that all together conform the bot configuration. It normally reads its configuration from a file (Freqtrade configuration file). -Per default, the bot loads configuration from the `config.json` file located in the current working directory. +Per default, the bot loads the configuration from the `config.json` file, located in the current working directory. -You can change the name of the configuration file used by the bot with the `-c/--config` command line option. +You can specify a different configuration file used by the bot with the `-c/--config` command line option. In some advanced use cases, multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream. @@ -22,19 +23,26 @@ The Freqtrade configuration file is to be written in the JSON format. Additionally to the standard JSON syntax, you may use one-line `// ...` and multi-line `/* ... */` comments in your configuration files and trailing commas in the lists of parameters. -Do not worry if you are not familiar with JSON format -- simply open the configuration file with an editor of your choice, make some changes to the parameters you need, save your changes and, finally, restart the bot or, if it was previously stopped, run it again with the changes you made to the configuration. The bot validates syntax of the configuration file at startup and will warn you if you made any errors editing it. +Do not worry if you are not familiar with JSON format -- simply open the configuration file with an editor of your choice, make some changes to the parameters you need, save your changes and, finally, restart the bot or, if it was previously stopped, run it again with the changes you made to the configuration. The bot validates syntax of the configuration file at startup and will warn you if you made any errors editing it, pointing out problematic lines. ## Configuration parameters The table below will list all configuration parameters available. -Mandatory parameters are marked as **Required**. +Freqtrade can also load many options via command line (CLI) arguments (check out the commands `--help` output for details). +The prevelance for all Options is as follows: + +- CLI arguments override any other option +- Configuration files are used in sequence (last file wins), and override Strategy configurations. +- Strategy configurations are only used if they are not set via configuration or via command line arguments. These options are market with [Strategy Override](#parameters-in-the-strategy) in the below table. + +Mandatory parameters are marked as **Required**, which means that they are required to be set in one of the possible ways. | Command | Default | Description | |----------|---------|-------------| | `max_open_trades` | 3 | **Required.** Number of trades open your bot will have. If -1 then it is ignored (i.e. potentially unlimited open trades) -| `stake_currency` | BTC | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy). -| `stake_amount` | 0.05 | **Required.** Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to `"unlimited"` to allow the bot to use all available balance. [Strategy Override](#parameters-in-the-strategy). +| `stake_currency` | BTC | **Required.** Crypto-currency used for trading. +| `stake_amount` | 0.05 | **Required.** Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to `"unlimited"` to allow the bot to use all available balance. | `amount_reserve_percent` | 0.05 | Reserve some amount in min pair stake amount. Default is 5%. The bot will reserve `amount_reserve_percent` + stop-loss value when calculating min pair stake amount in order to avoid possible trade refusals. | `ticker_interval` | [1m, 5m, 15m, 30m, 1h, 1d, ...] | The ticker interval to use (1min, 5 min, 15 min, 30 min, 1 hour or 1 day). Default is 5 minutes. [Strategy Override](#parameters-in-the-strategy). | `fiat_display_currency` | USD | **Required.** Fiat currency used to show your profits. More information below. @@ -99,8 +107,6 @@ Mandatory parameters are marked as **Required**. The following parameters can be set in either configuration file or strategy. Values set in the configuration file always overwrite values set in the strategy. -* `stake_currency` -* `stake_amount` * `ticker_interval` * `minimal_roi` * `stoploss` diff --git a/docs/data-download.md b/docs/data-download.md index c370b3919..244acb153 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -8,6 +8,10 @@ If no additional parameter is specified, freqtrade will download data for `"1m"` Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. +!!! Tip Updating existing data + If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. + Be carefull though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. + ### Pairs file In alternative to the whitelist from `config.json`, a `pairs.json` file can be used. diff --git a/docs/deprecated.md b/docs/deprecated.md index ed70b1936..349d41a09 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -4,7 +4,7 @@ This page contains description of the command line arguments, configuration para and the bot features that were declared as DEPRECATED by the bot development team and are no longer supported. Please avoid their usage in your configuration. -## Deprecated +## Removed features ### the `--refresh-pairs-cached` command line option @@ -12,9 +12,7 @@ and are no longer supported. Please avoid their usage in your configuration. Since this leads to much confusion, and slows down backtesting (while not being part of backtesting) this has been singled out as a seperate freqtrade subcommand `freqtrade download-data`. -This command line option was deprecated in `2019.7-dev` and will be removed after the next release. - -## Removed features +This command line option was deprecated in 2019.7-dev (develop branch) and removed in 2019.9 (master branch). ### The **--dynamic-whitelist** command line option diff --git a/docs/installation.md b/docs/installation.md index f15cc356c..081d7e0cf 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -99,8 +99,8 @@ sudo apt-get install build-essential git Before installing FreqTrade on a Raspberry Pi running the official Raspbian Image, make sure you have at least Python 3.6 installed. The default image only provides Python 3.5. Probably the easiest way to get a recent version of python is [miniconda](https://repo.continuum.io/miniconda/). -The following assumes that miniconda3 is installed and available in your environment. Last miniconda3 installation file use python 3.4, we will update to python 3.6 on this installation. -It's recommended to use (mini)conda for this as installation/compilation of `numpy`, `scipy` and `pandas` takes a long time. +The following assumes that miniconda3 is installed and available in your environment. Since the last miniconda3 installation file uses python 3.4, we will update to python 3.6 on this installation. +It's recommended to use (mini)conda for this as installation/compilation of `numpy` and `pandas` takes a long time. Additional package to install on your Raspbian, `libffi-dev` required by cryptography (from python-telegram-bot). @@ -109,13 +109,17 @@ conda config --add channels rpi conda install python=3.6 conda create -n freqtrade python=3.6 conda activate freqtrade -conda install scipy pandas numpy +conda install pandas numpy sudo apt install libffi-dev python3 -m pip install -r requirements-common.txt python3 -m pip install -e . ``` +!!! Note + This does not install hyperopt dependencies. To install these, please use `python3 -m pip install -e .[hyperopt]`. + We do not advise to run hyperopt on a Raspberry Pi, since this is a very resource-heavy operation, which should be done on powerful machine. + ### Common #### 1. Install TA-Lib @@ -175,7 +179,6 @@ cp config.json.example config.json ``` bash python3 -m pip install --upgrade pip -python3 -m pip install -r requirements.txt python3 -m pip install -e . ``` diff --git a/environment.yml b/environment.yml index cd3350fd5..4e8c1efcc 100644 --- a/environment.yml +++ b/environment.yml @@ -9,25 +9,26 @@ dependencies: - wheel - numpy - pandas - - scipy - SQLAlchemy - - scikit-learn - arrow - requests - urllib3 - wrapt - - joblib - jsonschema - tabulate - python-rapidjson - - filelock - flask - python-dotenv - cachetools - - scikit-optimize - python-telegram-bot # Optional for plotting - plotly + # Optional for hyperopt + - scipy + - scikit-optimize + - scikit-learn + - filelock + - joblib # Optional for development - flake8 - pytest diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 5ccc2ff3c..83fee0b0d 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,16 @@ """ FreqTrade bot """ -__version__ = '2019.7-dev' +__version__ = 'develop' + +if __version__ == 'develop': + + try: + import subprocess + __version__ = 'develop-' + subprocess.check_output( + ['git', 'log', '--format="%h"', '-n 1'], + stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"') + except Exception: + # git not available, ignore + pass class DependencyException(Exception): diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index bb961173b..6e2ecea2e 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -15,7 +15,7 @@ ARGS_STRATEGY = ["strategy", "strategy_path"] ARGS_MAIN = ARGS_COMMON + ARGS_STRATEGY + ["db_url", "sd_notify"] ARGS_COMMON_OPTIMIZE = ["ticker_interval", "timerange", - "max_open_trades", "stake_amount", "refresh_pairs"] + "max_open_trades", "stake_amount"] ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", "strategy_list", "export", "exportfilename"] diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index 61e862a9c..19c377732 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -27,6 +27,14 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool: logger.info("Checking exchange...") exchange = config.get('exchange', {}).get('name').lower() + if not exchange: + raise OperationalException( + f'This command requires a configured exchange. You should either use ' + f'`--exchange ` or specify a configuration file via `--config`.\n' + f'The following exchanges are supported by ccxt: ' + f'{", ".join(available_exchanges())}' + ) + if not is_exchange_available(exchange): raise OperationalException( f'Exchange "{exchange}" is not supported by ccxt ' diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index bf1ec3620..cb07dbdba 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -107,13 +107,6 @@ AVAILABLE_CLI_OPTIONS = { help='Specify stake_amount.', type=float, ), - "refresh_pairs": Arg( - '-r', '--refresh-pairs-cached', - help='Refresh the pairs files in tests/testdata with the latest data from the ' - 'exchange. Use it if you want to run your optimization commands with ' - 'up-to-date data.', - action='store_true', - ), # Backtesting "position_stacking": Arg( '--eps', '--enable-position-stacking', diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index df893b3ad..764593d0f 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -215,10 +215,6 @@ class Configuration: self._process_datadir_options(config) - self._args_to_config(config, argname='refresh_pairs', - logstring='Parameter -r/--refresh-pairs-cached detected ...', - deprecated_msg='-r/--refresh-pairs-cached will be removed soon.') - self._args_to_config(config, argname='strategy_list', logstring='Using strategy list of {} strategies', logfun=len) diff --git a/freqtrade/data/__init__.py b/freqtrade/data/__init__.py index 0a31d095c..0e7eea0d0 100644 --- a/freqtrade/data/__init__.py +++ b/freqtrade/data/__init__.py @@ -2,7 +2,7 @@ Module to handle data operations for freqtrade """ -# limit what's imported when using `from freqtrad.data import *`` +# limit what's imported when using `from freqtrade.data import *` __all__ = [ 'converter' ] diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index eb6ec0f2a..7d5e4540b 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -65,7 +65,6 @@ class DataProvider: """ return load_pair_history(pair=pair, ticker_interval=ticker_interval or self._config['ticker_interval'], - refresh_pairs=False, datadir=Path(self._config['datadir']) ) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index b4776fab0..865289f38 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -129,8 +129,7 @@ def load_pair_history(pair: str, else: logger.warning( f'No history data for pair: "{pair}", interval: {ticker_interval}. ' - 'Use --refresh-pairs-cached option or `freqtrade download-data` ' - 'script to download the data' + 'Use `freqtrade download-data` to download the data' ) return None @@ -142,33 +141,25 @@ def load_data(datadir: Path, exchange: Optional[Exchange] = None, timerange: TimeRange = TimeRange(None, None, 0, 0), fill_up_missing: bool = True, - live: bool = False ) -> Dict[str, DataFrame]: """ - Loads ticker history data for a list of pairs the given parameters + Loads ticker history data for a list of pairs :return: dict(:) + TODO: refresh_pairs is still used by edge to keep the data uptodate. + This should be replaced in the future. Instead, writing the current candles to disk + from dataprovider should be implemented, as this would avoid loading ohlcv data twice. + exchange and refresh_pairs are then not needed here nor in load_pair_history. """ result: Dict[str, DataFrame] = {} - if live: - if exchange: - logger.info('Live: Downloading data for all defined pairs ...') - exchange.refresh_latest_ohlcv([(pair, ticker_interval) for pair in pairs]) - result = {key[0]: value for key, value in exchange._klines.items() if value is not None} - else: - raise OperationalException( - "Exchange needs to be initialized when using live data." - ) - else: - logger.info('Using local backtesting data ...') - for pair in pairs: - hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, - datadir=datadir, timerange=timerange, - refresh_pairs=refresh_pairs, - exchange=exchange, - fill_up_missing=fill_up_missing) - if hist is not None: - result[pair] = hist + for pair in pairs: + hist = load_pair_history(pair=pair, ticker_interval=ticker_interval, + datadir=datadir, timerange=timerange, + refresh_pairs=refresh_pairs, + exchange=exchange, + fill_up_missing=fill_up_missing) + if hist is not None: + result[pair] = hist return result diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 7a3c290bf..3adf5eb43 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,9 +1,7 @@ import logging from typing import Any, Dict -from filelock import FileLock, Timeout - -from freqtrade import DependencyException, constants +from freqtrade import DependencyException, constants, OperationalException from freqtrade.state import RunMode from freqtrade.utils import setup_utils_configuration @@ -53,8 +51,12 @@ def start_hyperopt(args: Dict[str, Any]) -> None: :return: None """ # Import here to avoid loading hyperopt module when it's not used - from freqtrade.optimize.hyperopt import Hyperopt - + try: + from filelock import FileLock, Timeout + from freqtrade.optimize.hyperopt import Hyperopt + except ImportError as e: + raise OperationalException( + f"{e}. Please ensure that the hyperopt dependencies are installed.") from e # Initialize configuration config = setup_configuration(args, RunMode.HYPEROPT) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4956907fb..6074b281b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -414,8 +414,6 @@ class Backtesting: datadir=Path(self.config['datadir']), pairs=pairs, ticker_interval=self.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - exchange=self.exchange, timerange=timerange, ) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index f22bcb642..0cf5a009b 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -39,7 +39,8 @@ class EdgeCli: self.strategy = StrategyResolver(self.config).strategy self.edge = Edge(config, self.exchange, self.strategy) - self.edge._refresh_pairs = self.config.get('refresh_pairs', False) + # Set refresh_pairs to false for edge-cli (it must be true for edge) + self.edge._refresh_pairs = False self.timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c511aa5ac..a70ff8142 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -36,6 +36,11 @@ logger = logging.getLogger(__name__) INITIAL_POINTS = 30 + +# Keep no more than 2*SKOPT_MODELS_MAX_NUM models +# in the skopt models list +SKOPT_MODELS_MAX_NUM = 10 + MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization @@ -90,7 +95,7 @@ class Hyperopt: else: logger.debug('Ignoring max_open_trades (--disable-max-market-positions was used) ...') self.max_open_trades = 0 - self.position_stacking = self.config.get('position_stacking', False), + self.position_stacking = self.config.get('position_stacking', False) if self.has_space('sell'): # Make sure experimental is enabled @@ -255,12 +260,13 @@ class Hyperopt: spaces += self.custom_hyperopt.stoploss_space() return spaces - def generate_optimizer(self, _params: Dict) -> Dict: + def generate_optimizer(self, _params: Dict, iteration=None) -> Dict: """ Used Optimize function. Called once per epoch to optimize whatever is configured. Keep this function as optimized as possible! """ params = self.get_args(_params) + if self.has_space('roi'): self.backtesting.strategy.minimal_roi = \ self.custom_hyperopt.generate_roi_table(params) @@ -342,9 +348,26 @@ class Hyperopt: random_state=self.config.get('hyperopt_random_state', None) ) - def run_optimizer_parallel(self, parallel, asked) -> List: + def fix_optimizer_models_list(self): + """ + WORKAROUND: Since skopt is not actively supported, this resolves problems with skopt + memory usage, see also: https://github.com/scikit-optimize/scikit-optimize/pull/746 + + This may cease working when skopt updates if implementation of this intrinsic + part changes. + """ + n = len(self.opt.models) - SKOPT_MODELS_MAX_NUM + # Keep no more than 2*SKOPT_MODELS_MAX_NUM models in the skopt models list, + # remove the old ones. These are actually of no use, the current model + # from the estimator is the only one used in the skopt optimizer. + # Freqtrade code also does not inspect details of the models. + if n >= SKOPT_MODELS_MAX_NUM: + logger.debug(f"Fixing skopt models list, removing {n} old items...") + del self.opt.models[0:n] + + def run_optimizer_parallel(self, parallel, asked, i) -> List: return parallel(delayed( - wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) + wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) def load_previous_results(self): """ read trials file if we have one """ @@ -362,8 +385,6 @@ class Hyperopt: datadir=Path(self.config['datadir']), pairs=self.config['exchange']['pair_whitelist'], ticker_interval=self.backtesting.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - exchange=self.backtesting.exchange, timerange=timerange ) @@ -407,8 +428,9 @@ class Hyperopt: EVALS = max(self.total_epochs // jobs, 1) for i in range(EVALS): asked = self.opt.ask(n_points=jobs) - f_val = self.run_optimizer_parallel(parallel, asked) + f_val = self.run_optimizer_parallel(parallel, asked, i) self.opt.tell(asked, [v['loss'] for v in f_val]) + self.fix_optimizer_models_list() for j in range(jobs): current = i * jobs + j val = f_val[j] diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 5b2b08357..6ce5e888c 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -5,6 +5,7 @@ from typing import Any, Dict, List import arrow +from freqtrade import OperationalException from freqtrade.configuration import Configuration, TimeRange from freqtrade.configuration.directory_operations import create_userdata_dir from freqtrade.data.history import refresh_backtest_ohlcv_data @@ -70,6 +71,11 @@ def start_download_data(args: Dict[str, Any]) -> None: time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") timerange = TimeRange.parse_timerange(f'{time_since}-') + if 'pairs' not in config: + raise OperationalException( + "Downloading data requires a list of pairs. " + "Please check the documentation on how to configure this.") + dl_path = Path(config['datadir']) logger.info(f'About to download pairs: {config["pairs"]}, ' f'intervals: {config["timeframes"]} to {dl_path}') diff --git a/requirements-common.txt b/requirements-common.txt index 363890a4d..f10134203 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,28 +1,22 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.1159 +ccxt==1.18.1180 SQLAlchemy==1.3.8 -python-telegram-bot==12.1.0 +python-telegram-bot==12.1.1 arrow==0.15.2 cachetools==3.1.1 requests==2.22.0 -urllib3==1.25.3 +urllib3==1.25.5 wrapt==1.11.2 -scikit-learn==0.21.3 -joblib==0.13.2 jsonschema==3.0.2 TA-Lib==0.4.17 tabulate==0.8.3 coinmarketcap==5.0.3 -# Required for hyperopt -scikit-optimize==0.5.2 -filelock==3.0.12 - # find first, C search in arrays py_find_1st==1.1.4 -#Load ticker files 30% faster +# Load ticker files 30% faster python-rapidjson==0.8.0 # Notify systemd diff --git a/requirements-dev.txt b/requirements-dev.txt index 1b9bf7570..dcf2c7217 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,13 +1,14 @@ # Include all requirements to run the bot. -r requirements.txt -r requirements-plot.txt +-r requirements-hyperopt.txt coveralls==1.8.2 flake8==3.7.8 flake8-type-annotations==0.1.0 flake8-tidy-imports==2.0.0 mypy==0.720 -pytest==5.1.2 +pytest==5.1.3 pytest-asyncio==0.10.0 pytest-cov==2.7.1 pytest-mock==1.10.4 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt new file mode 100644 index 000000000..bb0ad60f0 --- /dev/null +++ b/requirements-hyperopt.txt @@ -0,0 +1,9 @@ +# Include all requirements to run the bot. +# -r requirements.txt + +# Required for hyperopt +scipy==1.3.1 +scikit-learn==0.21.3 +scikit-optimize==0.5.2 +filelock==3.0.12 +joblib==0.13.2 diff --git a/requirements.txt b/requirements.txt index 9a723fee4..2767180ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ numpy==1.17.2 pandas==0.25.1 -scipy==1.3.1 diff --git a/setup.py b/setup.py index ca94dd338..2eb09e589 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,13 @@ if readme_file.is_file(): # Requirements used for submodules api = ['flask'] plot = ['plotly>=4.0'] +hyperopt = [ + 'scipy', + 'scikit-learn', + 'scikit-optimize', + 'filelock', + 'joblib', + ] develop = [ 'coveralls', @@ -38,7 +45,7 @@ jupyter = [ 'ipykernel', ] -all_extra = api + plot + develop + jupyter +all_extra = api + plot + develop + jupyter + hyperopt setup(name='freqtrade', version=__version__, @@ -62,14 +69,10 @@ setup(name='freqtrade', 'requests', 'urllib3', 'wrapt', - 'scikit-learn', - 'joblib', 'jsonschema', 'TA-Lib', 'tabulate', 'coinmarketcap', - 'scikit-optimize', - 'filelock', 'py_find_1st', 'python-rapidjson', 'sdnotify', @@ -77,15 +80,14 @@ setup(name='freqtrade', # from requirements.txt 'numpy', 'pandas', - 'scipy', ], extras_require={ 'api': api, 'dev': all_extra, 'plot': plot, - 'all': all_extra, 'jupyter': jupyter, - + 'hyperopt': hyperopt, + 'all': all_extra, }, include_package_data=True, zip_safe=False, diff --git a/tests/conftest.py b/tests/conftest.py index 8a5ba6683..6a0a74b5b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1054,3 +1054,24 @@ def rpc_balance(): def testdatadir() -> Path: """Return the path where testdata files are stored""" return (Path(__file__).parent / "testdata").resolve() + + +@pytest.fixture(scope="function") +def import_fails() -> None: + # Source of this test-method: + # https://stackoverflow.com/questions/2481511/mocking-importerror-in-python + import builtins + realimport = builtins.__import__ + + def mockedimport(name, *args, **kwargs): + if name in ["filelock"]: + raise ImportError(f"No module named '{name}'") + return realimport(name, *args, **kwargs) + + builtins.__import__ = mockedimport + + # Run test - then cleanup + yield + + # restore previous importfunction + builtins.__import__ = realimport diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 72da47e76..e773a970e 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -24,7 +24,6 @@ def test_parse_ticker_dataframe(ticker_history_list, caplog): def test_ohlcv_fill_up_missing_data(testdatadir, caplog): data = load_pair_history(datadir=testdatadir, ticker_interval='1m', - refresh_pairs=False, pair='UNITTEST/BTC', fill_up_missing=False) caplog.set_level(logging.DEBUG) diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index ec176d889..39e2f7d2e 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -45,7 +45,6 @@ def test_historic_ohlcv(mocker, default_conf, ticker_history): data = dp.historic_ohlcv("UNITTEST/BTC", "5m") assert isinstance(data, DataFrame) assert historymock.call_count == 1 - assert historymock.call_args_list[0][1]["refresh_pairs"] is False assert historymock.call_args_list[0][1]["ticker_interval"] == "5m" diff --git a/tests/data/test_history.py b/tests/data/test_history.py index e747794e5..e386c3506 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -74,8 +74,7 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf, testdatadir) -> Non assert ld is None assert log_has( 'No history data for pair: "UNITTEST/BTC", interval: 7m. ' - 'Use --refresh-pairs-cached option or `freqtrade download-data` ' - 'script to download the data', caplog + 'Use `freqtrade download-data` to download the data', caplog ) @@ -105,13 +104,11 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, # do not download a new pair if refresh_pairs isn't set history.load_pair_history(datadir=testdatadir, ticker_interval='1m', - refresh_pairs=False, pair='MEME/BTC') assert os.path.isfile(file) is False assert log_has( 'No history data for pair: "MEME/BTC", interval: 1m. ' - 'Use --refresh-pairs-cached option or `freqtrade download-data` ' - 'script to download the data', caplog + 'Use `freqtrade download-data` to download the data', caplog ) # download a new pair if refresh_pairs is set @@ -134,31 +131,6 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, _clean_test_file(file) -def test_load_data_live(default_conf, mocker, caplog, testdatadir) -> None: - refresh_mock = MagicMock() - mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock) - exchange = get_patched_exchange(mocker, default_conf) - - history.load_data(datadir=testdatadir, ticker_interval='5m', - pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'], - live=True, - exchange=exchange) - assert refresh_mock.call_count == 1 - assert len(refresh_mock.call_args_list[0][0][0]) == 2 - assert log_has('Live: Downloading data for all defined pairs ...', caplog) - - -def test_load_data_live_noexchange(default_conf, mocker, caplog, testdatadir) -> None: - - with pytest.raises(OperationalException, - match=r'Exchange needs to be initialized when using live data.'): - history.load_data(datadir=testdatadir, ticker_interval='5m', - pairs=['UNITTEST/BTC', 'UNITTEST2/BTC'], - exchange=None, - live=True, - ) - - def test_testdata_path(testdatadir) -> None: assert str(Path('tests') / 'testdata') in str(testdatadir) @@ -349,7 +321,6 @@ def test_load_partial_missing(testdatadir, caplog) -> None: start = arrow.get('2018-01-01T00:00:00') end = arrow.get('2018-01-11T00:00:00') tickerdata = history.load_data(testdatadir, '5m', ['UNITTEST/BTC'], - refresh_pairs=False, timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) # timedifference in 5 minutes @@ -364,7 +335,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None: start = arrow.get('2018-01-10T00:00:00') end = arrow.get('2018-02-20T00:00:00') tickerdata = history.load_data(datadir=testdatadir, ticker_interval='5m', - pairs=['UNITTEST/BTC'], refresh_pairs=False, + pairs=['UNITTEST/BTC'], timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) # timedifference in 5 minutes diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 7b50f2b18..fa40809d8 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -188,16 +188,12 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'position_stacking' not in config assert not log_has('Parameter --enable-position-stacking detected ...', caplog) - assert 'refresh_pairs' not in config - assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' not in config assert 'export' not in config assert 'runmode' in config assert config['runmode'] == RunMode.BACKTEST -@pytest.mark.filterwarnings("ignore:DEPRECATED") def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) mocker.patch( @@ -213,7 +209,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--ticker-interval', '1m', '--enable-position-stacking', '--disable-max-market-positions', - '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo', '--export-filename', 'foo_bar.json' @@ -240,9 +235,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert log_has('Parameter --disable-max-market-positions detected ...', caplog) assert log_has('max_open_trades set to unlimited ...', caplog) - assert 'refresh_pairs' in config - assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' in config assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index f312bba2c..97103da55 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -3,8 +3,6 @@ from unittest.mock import MagicMock -import pytest - from freqtrade.edge import PairInfo from freqtrade.optimize import setup_configuration, start_edge from freqtrade.optimize.edge_cli import EdgeCli @@ -35,14 +33,10 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'ticker_interval' in config assert not log_has_re('Parameter -i/--ticker-interval detected .*', caplog) - assert 'refresh_pairs' not in config - assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' not in config assert 'stoploss_range' not in config -@pytest.mark.filterwarnings("ignore:DEPRECATED") def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> None: patched_configuration_load_config_file(mocker, edge_conf) mocker.patch( @@ -56,7 +50,6 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N '--datadir', '/foo/bar', 'edge', '--ticker-interval', '1m', - '--refresh-pairs-cached', '--timerange', ':100', '--stoplosses=-0.01,-0.10,-0.001' ] @@ -74,8 +67,6 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', caplog) - assert 'refresh_pairs' in config - assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) assert 'timerange' in config assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 2e383c839..55f94f572 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -86,15 +86,11 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca assert 'position_stacking' not in config assert not log_has('Parameter --enable-position-stacking detected ...', caplog) - assert 'refresh_pairs' not in config - assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' not in config assert 'runmode' in config assert config['runmode'] == RunMode.HYPEROPT -@pytest.mark.filterwarnings("ignore:DEPRECATED") def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) mocker.patch( @@ -108,7 +104,6 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo 'hyperopt', '--ticker-interval', '1m', '--timerange', ':100', - '--refresh-pairs-cached', '--enable-position-stacking', '--disable-max-market-positions', '--epochs', '1000', @@ -137,9 +132,6 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo assert log_has('Parameter --disable-max-market-positions detected ...', caplog) assert log_has('max_open_trades set to unlimited ...', caplog) - assert 'refresh_pairs' in config - assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' in config assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) @@ -198,6 +190,24 @@ def test_hyperoptlossresolver_wrongname(mocker, default_conf, caplog) -> None: HyperOptLossResolver(default_conf, ).hyperopt +def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None: + start_mock = MagicMock() + patched_configuration_load_config_file(mocker, default_conf) + + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) + patch_exchange(mocker) + + args = [ + '--config', 'config.json', + 'hyperopt', + '--epochs', '5' + ] + args = get_args(args) + + with pytest.raises(OperationalException, match=r"Please ensure that the hyperopt dependencies"): + start_hyperopt(args) + + def test_start(mocker, default_conf, caplog) -> None: start_mock = MagicMock() patched_configuration_load_config_file(mocker, default_conf) diff --git a/tests/test_arguments.py b/tests/test_arguments.py index 7b18aa356..cf0104c01 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -99,7 +99,6 @@ def test_parse_args_backtesting_custom() -> None: '-c', 'test_conf.json', 'backtesting', '--ticker-interval', '1m', - '--refresh-pairs-cached', '--strategy-list', 'DefaultStrategy', 'SampleStrategy' @@ -110,7 +109,6 @@ def test_parse_args_backtesting_custom() -> None: assert call_args["subparser"] == 'backtesting' assert call_args["func"] is not None assert call_args["ticker_interval"] == '1m' - assert call_args["refresh_pairs"] is True assert type(call_args["strategy_list"]) is list assert len(call_args["strategy_list"]) == 2 diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 2cf273091..330b82d39 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -358,14 +358,10 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert 'position_stacking' not in config assert not log_has('Parameter --enable-position-stacking detected ...', caplog) - assert 'refresh_pairs' not in config - assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) - assert 'timerange' not in config assert 'export' not in config -@pytest.mark.filterwarnings("ignore:DEPRECATED") def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) mocker.patch( @@ -385,7 +381,6 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--ticker-interval', '1m', '--enable-position-stacking', '--disable-max-market-positions', - '--refresh-pairs-cached', '--timerange', ':100', '--export', '/bar/foo' ] @@ -415,8 +410,6 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non assert log_has('Parameter --disable-max-market-positions detected ...', caplog) assert log_has('max_open_trades set to unlimited ...', caplog) - assert 'refresh_pairs'in config - assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog) assert 'timerange' in config assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) @@ -546,6 +539,13 @@ def test_check_exchange(default_conf, caplog) -> None: default_conf['runmode'] = RunMode.PLOT assert check_exchange(default_conf) + # Test no exchange... + default_conf.get('exchange').update({'name': ''}) + default_conf['runmode'] = RunMode.OTHER + with pytest.raises(OperationalException, + match=r'This command requires a configured exchange.*'): + check_exchange(default_conf) + def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) diff --git a/tests/test_utils.py b/tests/test_utils.py index 386efb5ec..c99044610 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,12 +1,14 @@ import re +from pathlib import Path from unittest.mock import MagicMock, PropertyMock import pytest +from freqtrade import OperationalException from freqtrade.state import RunMode -from tests.conftest import get_args, log_has, patch_exchange from freqtrade.utils import (setup_utils_configuration, start_create_userdir, start_download_data, start_list_exchanges) +from tests.conftest import get_args, log_has, patch_exchange def test_setup_utils_configuration(): @@ -103,3 +105,42 @@ def test_download_data_no_markets(mocker, caplog): start_download_data(get_args(args)) assert dl_mock.call_args[1]['timerange'].starttype == "date" assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) + + +def test_download_data_no_exchange(mocker, caplog): + mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + args = [ + "download-data", + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, + match=r"This command requires a configured exchange.*"): + start_download_data(pargs) + + +def test_download_data_no_pairs(mocker, caplog): + + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + + mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + args = [ + "download-data", + "--exchange", + "binance", + ] + pargs = get_args(args) + pargs['config'] = None + with pytest.raises(OperationalException, + match=r"Downloading data requires a list of pairs\..*"): + start_download_data(pargs)