diff --git a/.dependabot/config.yml b/.dependabot/config.yml new file mode 100644 index 000000000..66b91e99f --- /dev/null +++ b/.dependabot/config.yml @@ -0,0 +1,17 @@ +version: 1 + +update_configs: + - package_manager: "python" + directory: "/" + update_schedule: "weekly" + allowed_updates: + - match: + update_type: "all" + target_branch: "develop" + + - package_manager: "docker" + directory: "/" + update_schedule: "daily" + allowed_updates: + - match: + update_type: "all" diff --git a/.pyup.yml b/.pyup.yml deleted file mode 100644 index b1b721113..000000000 --- a/.pyup.yml +++ /dev/null @@ -1,37 +0,0 @@ -# autogenerated pyup.io config file -# see https://pyup.io/docs/configuration/ for all available options - -# configure updates globally -# default: all -# allowed: all, insecure, False -update: all - -# configure dependency pinning globally -# default: True -# allowed: True, False -pin: True - -# update schedule -# default: empty -# allowed: "every day", "every week", .. -schedule: "every week" - - -search: False -# Specify requirement files by hand, default is empty -# default: empty -# allowed: list -requirements: - - requirements.txt - - requirements-dev.txt - - requirements-plot.txt - - requirements-common.txt - - -# configure the branch prefix the bot is using -# default: pyup- -branch_prefix: pyup/ - -# allow to close stale PRs -# default: True -close_prs: True diff --git a/Dockerfile b/Dockerfile index 7a0298719..8677b54de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7.3-slim-stretch +FROM python:3.7.4-slim-stretch RUN apt-get update \ && apt-get -y install curl build-essential libssl-dev \ diff --git a/docs/backtesting.md b/docs/backtesting.md index 7e9f7ff53..90256283e 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -3,9 +3,43 @@ This page explains how to validate your strategy performance by using Backtesting. +## Getting data for backtesting and hyperopt + +To download data (candles / OHLCV) needed for backtesting and hyperoptimization use the `freqtrade download-data` command. + +If no additional parameter is specified, freqtrade will download data for `"1m"` and `"5m"` timeframes. +Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. + +Alternatively, a `pairs.json` file can be used. + +If you are using Binance for example: + +- create a directory `user_data/data/binance` and copy `pairs.json` in that directory. +- update the `pairs.json` to contain the currency pairs you are interested in. + +```bash +mkdir -p user_data/data/binance +cp freqtrade/tests/testdata/pairs.json user_data/data/binance +``` + +Then run: + +```bash +freqtrade download-data --exchange binance +``` + +This will download ticker data for all the currency pairs you defined in `pairs.json`. + +- To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. +- To change the exchange used to download the tickers, please use a different configuration file (you'll probably need to adjust ratelimits etc.) +- To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. +- To download ticker data for only 10 days, use `--days 10` (defaults to 30 days). +- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. +- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. + ## Test your strategy with Backtesting -Now you have good Buy and Sell strategies, you want to test it against +Now you have good Buy and Sell strategies and some historic data, you want to test it against real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). @@ -109,37 +143,6 @@ The full timerange specification: - Use tickframes between POSIX timestamps 1527595200 1527618600: `--timerange=1527595200-1527618600` -#### Downloading new set of ticker data - -To download new set of backtesting ticker data, you can use a download script. - -If you are using Binance for example: - -- create a directory `user_data/data/binance` and copy `pairs.json` in that directory. -- update the `pairs.json` to contain the currency pairs you are interested in. - -```bash -mkdir -p user_data/data/binance -cp freqtrade/tests/testdata/pairs.json user_data/data/binance -``` - -Then run: - -```bash -python scripts/download_backtest_data.py --exchange binance -``` - -This will download ticker data for all the currency pairs you defined in `pairs.json`. - -- To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. -- To change the exchange used to download the tickers, use `--exchange`. Default is `bittrex`. -- To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. -- To download ticker data for only 10 days, use `--days 10`. -- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. -- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with other options. - -For help about backtesting usage, please refer to [Backtesting commands](#backtesting-commands). - ## Understand the backtesting result The most important in the backtesting is to understand the result. diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 0c6dcd7f4..2873f5e8f 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -2,7 +2,7 @@ This page explains the different parameters of the bot and how to run it. -!Note: +!!! Note: If you've used `setup.sh`, don't forget to activate your virtual environment (`source .env/bin/activate`) before running freqtrade commands. @@ -43,20 +43,23 @@ optional arguments: --sd-notify Notify systemd service manager. ``` -### How to use a different configuration file? +### How to specify which configuration file be used? -The bot allows you to select which configuration file you want to use. Per -default, the bot will load the file `./config.json` +The bot allows you to select which configuration file you want to use by means of +the `-c/--config` command line option: ```bash freqtrade -c path/far/far/away/config.json ``` +Per default, the bot loads the `config.json` configuration file from the current +working directory. + ### How to use multiple configuration files? The bot allows you to use multiple configuration files by specifying multiple -`-c/--config` configuration options in the command line. Configuration parameters -defined in latter configuration files override parameters with the same name +`-c/--config` options in the command line. Configuration parameters +defined in the latter configuration files override parameters with the same name defined in the previous configuration files specified in the command line earlier. For example, you can make a separate configuration file with your key and secrete @@ -181,19 +184,11 @@ optional arguments: result.json) ``` -### How to use **--refresh-pairs-cached** parameter? +### Getting historic data for backtesting -The first time your run Backtesting, it will take the pairs you have -set in your config file and download data from the Exchange. - -If for any reason you want to update your data set, you use -`--refresh-pairs-cached` to force Backtesting to update the data it has. - -!!! Note - Use it only if you want to update your data set. You will not be able to come back to the previous version. - -To test your strategy with latest data, we recommend continuing using -the parameter `-l` or `--live`. +The first time your run Backtesting, you will need to download some historic data first. +This can be accomplished by using `freqtrade download-data`. +Check the corresponding [help page section](backtesting.md#Getting-data-for-backtesting-and-hyperopt) for more details ## Hyperopt commands @@ -269,7 +264,7 @@ optional arguments: ## Edge commands -To know your trade expectacny and winrate against historical data, you can use Edge. +To know your trade expectancy and winrate against historical data, you can use Edge. ``` usage: freqtrade edge [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE] diff --git a/docs/configuration.md b/docs/configuration.md index fc450bd82..c277e1c06 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,15 +1,34 @@ # Configure the bot -This page explains how to configure your `config.json` file. +This page explains how to configure the bot. -## Setup config.json +## The Freqtrade configuration file -We recommend to copy and use the `config.json.example` as a template +The bot uses a set of configuration parameters during its operation that all together conform the bot configuration. It normally reads its configuration from a file (Freqtrade configuration file). + +Per default, the bot loads configuration from the `config.json` file located in the current working directory. + +You can change the name of the configuration file used by the bot with the `-c/--config` command line option. + +In some advanced use cases, multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream. + +If you used the [Quick start](installation.md/#quick-start) method for installing +the bot, the installation script should have already created the default configuration file (`config.json`) for you. + +If default configuration file is not created we recommend you to copy and use the `config.json.example` as a template for your bot configuration. -The table below will list all configuration parameters. +The Freqtrade configuration file is to be written in the JSON format. -Mandatory Parameters are marked as **Required**. +Additionally to the standard JSON syntax, you may use one-line `// ...` and multi-line `/* ... */` comments in your configuration files and trailing commas in the lists of parameters. + +Do not worry if you are not familiar with JSON format -- simply open the configuration file with an editor of your choice, make some changes to the parameters you need, save your changes and, finally, restart the bot or, if it was previously stopped, run it again with the changes you made to the configuration. The bot validates syntax of the configuration file at startup and will warn you if you made any errors editing it. + +## Configuration parameters + +The table below will list all configuration parameters available. + +Mandatory parameters are marked as **Required**. | Command | Default | Description | |----------|---------|-------------| diff --git a/docs/docker.md b/docs/docker.md index 615d31796..923dec1e2 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -26,6 +26,10 @@ To update the image, simply run the above commands again and restart your runnin Should you require additional libraries, please [build the image yourself](#build-your-own-docker-image). +!!! Note Docker image update frequency + The official docker images with tags `master`, `develop` and `latest` are automatically rebuild once a week to keep the base image uptodate. + In addition to that, every merge to `develop` will trigger a rebuild for `develop` and `latest`. + ### Prepare the configuration files Even though you will use docker, you'll still need some files from the github repository. diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 15f44955b..d71ebfded 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -274,27 +274,24 @@ Please always check the mode of operation to select the correct method to get da #### Possible options for DataProvider -- `available_pairs` - Property with tuples listing cached pairs with their intervals. (pair, interval) -- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for all pairs in the whitelist, returns DataFrame or empty DataFrame -- `historic_ohlcv(pair, ticker_interval)` - Data stored on disk +- `available_pairs` - Property with tuples listing cached pairs with their intervals (pair, interval). +- `ohlcv(pair, ticker_interval)` - Currently cached ticker data for the pair, returns DataFrame or empty DataFrame. +- `historic_ohlcv(pair, ticker_interval)` - Returns historical data stored on disk. +- `get_pair_dataframe(pair, ticker_interval)` - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). - `runmode` - Property containing the current runmode. -#### ohlcv / historic_ohlcv +#### Example: fetch live ohlcv / historic data for the first informative pair ``` python if self.dp: - if self.dp.runmode in ('live', 'dry_run'): - if (f'{self.stake_currency}/BTC', self.ticker_interval) in self.dp.available_pairs: - data_eth = self.dp.ohlcv(pair='{self.stake_currency}/BTC', - ticker_interval=self.ticker_interval) - else: - # Get historic ohlcv data (cached on disk). - history_eth = self.dp.historic_ohlcv(pair='{self.stake_currency}/BTC', - ticker_interval='1h') + inf_pair, inf_timeframe = self.informative_pairs()[0] + informative = self.dp.get_pair_dataframe(pair=inf_pair, + ticker_interval=inf_timeframe) ``` !!! Warning Warning about backtesting - Be carefull when using dataprovider in backtesting. `historic_ohlcv()` provides the full time-range in one go, + Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` + for the backtesting runmode) provides the full time-range in one go, so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). !!! Warning Warning in hyperopt @@ -309,8 +306,10 @@ if self.dp: dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] ``` -!Warning The order book is not part of the historic data which means backtesting and hyperopt will not work if this - method is used. + +!!! Warning + The order book is not part of the historic data which means backtesting and hyperopt will not work if this + method is used. #### Available Pairs diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 548b508a7..ac59421a7 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,2 +1,4 @@ -from freqtrade.configuration.arguments import Arguments, TimeRange # noqa: F401 +from freqtrade.configuration.arguments import Arguments # noqa: F401 +from freqtrade.configuration.timerange import TimeRange # noqa: F401 from freqtrade.configuration.configuration import Configuration # noqa: F401 +from freqtrade.configuration.config_validation import validate_config_consistency # noqa: F401 diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index dd5a4290e..c45e3d7ba 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -2,10 +2,8 @@ This module contains the argument manager class """ import argparse -import re -from typing import List, NamedTuple, Optional +from typing import List, Optional -import arrow from freqtrade.configuration.cli_options import AVAILABLE_CLI_OPTIONS from freqtrade import constants @@ -24,7 +22,7 @@ ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_pos ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "position_stacking", "epochs", "spaces", "use_max_market_positions", "print_all", - "print_colorized", "hyperopt_jobs", + "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", "hyperopt_continue", "hyperopt_loss"] @@ -32,7 +30,7 @@ ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] ARGS_LIST_EXCHANGES = ["print_one_column"] -ARGS_DOWNLOADER = ARGS_COMMON + ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] +ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", @@ -42,17 +40,7 @@ ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + ARGS_PLOT_PROFIT = (ARGS_COMMON + ARGS_STRATEGY + ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source"]) - -class TimeRange(NamedTuple): - """ - NamedTuple defining timerange inputs. - [start/stop]type defines if [start/stop]ts shall be used. - if *type is None, don't use corresponding startvalue. - """ - starttype: Optional[str] = None - stoptype: Optional[str] = None - startts: int = 0 - stopts: int = 0 +NO_CONF_REQURIED = ["start_download_data"] class Arguments(object): @@ -89,7 +77,10 @@ class Arguments(object): # Workaround issue in argparse with action='append' and default value # (see https://bugs.python.org/issue16399) - if not self._no_default_config and parsed_arg.config is None: + # Allow no-config for certain commands (like downloading / plotting) + if (not self._no_default_config and parsed_arg.config is None + and not (hasattr(parsed_arg, 'func') + and parsed_arg.func.__name__ in NO_CONF_REQURIED)): parsed_arg.config = [constants.DEFAULT_CONFIG] return parsed_arg @@ -107,7 +98,7 @@ class Arguments(object): :return: None """ from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge - from freqtrade.utils import start_list_exchanges + from freqtrade.utils import start_download_data, start_list_exchanges subparsers = self.parser.add_subparsers(dest='subparser') @@ -134,44 +125,10 @@ class Arguments(object): list_exchanges_cmd.set_defaults(func=start_list_exchanges) self._build_args(optionlist=ARGS_LIST_EXCHANGES, parser=list_exchanges_cmd) - @staticmethod - def parse_timerange(text: Optional[str]) -> TimeRange: - """ - Parse the value of the argument --timerange to determine what is the range desired - :param text: value from --timerange - :return: Start and End range period - """ - if text is None: - return TimeRange(None, None, 0, 0) - syntax = [(r'^-(\d{8})$', (None, 'date')), - (r'^(\d{8})-$', ('date', None)), - (r'^(\d{8})-(\d{8})$', ('date', 'date')), - (r'^-(\d{10})$', (None, 'date')), - (r'^(\d{10})-$', ('date', None)), - (r'^(\d{10})-(\d{10})$', ('date', 'date')), - (r'^(-\d+)$', (None, 'line')), - (r'^(\d+)-$', ('line', None)), - (r'^(\d+)-(\d+)$', ('index', 'index'))] - for rex, stype in syntax: - # Apply the regular expression to text - match = re.match(rex, text) - if match: # Regex has matched - rvals = match.groups() - index = 0 - start: int = 0 - stop: int = 0 - if stype[0]: - starts = rvals[index] - if stype[0] == 'date' and len(starts) == 8: - start = arrow.get(starts, 'YYYYMMDD').timestamp - else: - start = int(starts) - index += 1 - if stype[1]: - stops = rvals[index] - if stype[1] == 'date' and len(stops) == 8: - stop = arrow.get(stops, 'YYYYMMDD').timestamp - else: - stop = int(stops) - return TimeRange(stype[0], stype[1], start, stop) - raise Exception('Incorrect syntax for timerange "%s"' % text) + # Add download-data subcommand + download_data_cmd = subparsers.add_parser( + 'download-data', + help='Download backtesting data.' + ) + download_data_cmd.set_defaults(func=start_download_data) + self._build_args(optionlist=ARGS_DOWNLOAD_DATA, parser=download_data_cmd) diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index b098fa8bc..d39013737 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -198,6 +198,12 @@ AVAILABLE_CLI_OPTIONS = { action='store_false', default=True, ), + "print_json": Arg( + '--print-json', + help='Print best result detailization in JSON format.', + action='store_true', + default=False, + ), "hyperopt_jobs": Arg( '-j', '--job-workers', help='The number of concurrently running jobs for hyperoptimization ' @@ -248,7 +254,8 @@ AVAILABLE_CLI_OPTIONS = { # Script options "pairs": Arg( '-p', '--pairs', - help='Show profits for only these pairs. Pairs are comma-separated.', + help='Show profits for only these pairs. Pairs are space-separated.', + nargs='+', ), # Download data "pairs_file": Arg( @@ -270,9 +277,10 @@ AVAILABLE_CLI_OPTIONS = { "timeframes": Arg( '-t', '--timeframes', help=f'Specify which tickers to download. Space-separated list. ' - f'Default: `{constants.DEFAULT_DOWNLOAD_TICKER_INTERVALS}`.', + f'Default: `1m 5m`.', choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h', '1d', '3d', '1w'], + default=['1m', '5m'], nargs='+', ), "erase": Arg( diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py new file mode 100644 index 000000000..92846b704 --- /dev/null +++ b/freqtrade/configuration/config_validation.py @@ -0,0 +1,102 @@ +import logging +from typing import Any, Dict + +from jsonschema import Draft4Validator, validators +from jsonschema.exceptions import ValidationError, best_match + +from freqtrade import constants, OperationalException + + +logger = logging.getLogger(__name__) + + +def _extend_validator(validator_class): + """ + Extended validator for the Freqtrade configuration JSON Schema. + Currently it only handles defaults for subschemas. + """ + validate_properties = validator_class.VALIDATORS['properties'] + + def set_defaults(validator, properties, instance, schema): + for prop, subschema in properties.items(): + if 'default' in subschema: + instance.setdefault(prop, subschema['default']) + + for error in validate_properties( + validator, properties, instance, schema, + ): + yield error + + return validators.extend( + validator_class, {'properties': set_defaults} + ) + + +FreqtradeValidator = _extend_validator(Draft4Validator) + + +def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: + """ + Validate the configuration follow the Config Schema + :param conf: Config in JSON format + :return: Returns the config if valid, otherwise throw an exception + """ + try: + FreqtradeValidator(constants.CONF_SCHEMA).validate(conf) + return conf + except ValidationError as e: + logger.critical( + f"Invalid configuration. See config.json.example. Reason: {e}" + ) + raise ValidationError( + best_match(Draft4Validator(constants.CONF_SCHEMA).iter_errors(conf)).message + ) + + +def validate_config_consistency(conf: Dict[str, Any]) -> None: + """ + Validate the configuration consistency. + Should be ran after loading both configuration and strategy, + since strategies can set certain configuration settings too. + :param conf: Config in JSON format + :return: Returns None if everything is ok, otherwise throw an OperationalException + """ + # validating trailing stoploss + _validate_trailing_stoploss(conf) + _validate_edge(conf) + + +def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None: + + # Skip if trailing stoploss is not activated + if not conf.get('trailing_stop', False): + return + + tsl_positive = float(conf.get('trailing_stop_positive', 0)) + tsl_offset = float(conf.get('trailing_stop_positive_offset', 0)) + tsl_only_offset = conf.get('trailing_only_offset_is_reached', False) + + if tsl_only_offset: + if tsl_positive == 0.0: + raise OperationalException( + f'The config trailing_only_offset_is_reached needs ' + 'trailing_stop_positive_offset to be more than 0 in your config.') + if tsl_positive > 0 and 0 < tsl_offset <= tsl_positive: + raise OperationalException( + f'The config trailing_stop_positive_offset needs ' + 'to be greater than trailing_stop_positive_offset in your config.') + + +def _validate_edge(conf: Dict[str, Any]) -> None: + """ + Edge and Dynamic whitelist should not both be enabled, since edge overrides dynamic whitelists. + """ + + if not conf.get('edge', {}).get('enabled'): + return + + if conf.get('pairlist', {}).get('method') == 'VolumePairList': + raise OperationalException( + "Edge and VolumePairList are incompatible, " + "Edge will override whatever pairs VolumePairlist selects." + ) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 587c8757e..329058cef 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -4,15 +4,17 @@ This module contains the configuration class import logging import warnings from argparse import Namespace +from pathlib import Path from typing import Any, Callable, Dict, List, Optional -from freqtrade import OperationalException, constants +from freqtrade import constants, OperationalException from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.create_datadir import create_datadir -from freqtrade.configuration.json_schema import validate_config_schema +from freqtrade.configuration.config_validation import (validate_config_schema, + validate_config_consistency) from freqtrade.configuration.load_config import load_config_file from freqtrade.loggers import setup_logging -from freqtrade.misc import deep_merge_dicts +from freqtrade.misc import deep_merge_dicts, json_load from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -52,6 +54,9 @@ class Configuration(object): # Keep this method as staticmethod, so it can be used from interactive environments config: Dict[str, Any] = {} + if not files: + return constants.MINIMAL_CONFIG.copy() + # We expect here a list of config filenames for path in files: logger.info(f'Using config: {path} ...') @@ -77,8 +82,6 @@ class Configuration(object): # Load all configs config: Dict[str, Any] = Configuration.from_files(self.args.config) - self._validate_config_consistency(config) - self._process_common_options(config) self._process_optimize_options(config) @@ -87,6 +90,13 @@ class Configuration(object): self._process_runmode(config) + # Check if the exchange set by the user is supported + check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) + + self._resolve_pairs_list(config) + + validate_config_consistency(config) + return config def _process_logging_options(self, config: Dict[str, Any]) -> None: @@ -147,9 +157,6 @@ class Configuration(object): if 'sd_notify' in self.args and self.args.sd_notify: config['internals'].update({'sd_notify': True}) - # Check if the exchange set by the user is supported - check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) - def _process_datadir_options(self, config: Dict[str, Any]) -> None: """ Extract information for sys.argv and load datadir configuration: @@ -242,6 +249,9 @@ class Configuration(object): else: config.update({'print_colorized': True}) + self._args_to_config(config, argname='print_json', + logstring='Parameter --print-json detected ...') + self._args_to_config(config, argname='hyperopt_jobs', logstring='Parameter -j/--job-workers detected: {}') @@ -273,44 +283,28 @@ class Configuration(object): self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') + self._args_to_config(config, argname='erase', + logstring='Erase detected. Deleting existing data.') + + self._args_to_config(config, argname='timeframes', + logstring='timeframes --timeframes: {}') + + self._args_to_config(config, argname='days', + logstring='Detected --days: {}') + + if "exchange" in self.args and self.args.exchange: + config['exchange']['name'] = self.args.exchange + logger.info(f"Using exchange {config['exchange']['name']}") + def _process_runmode(self, config: Dict[str, Any]) -> None: if not self.runmode: # Handle real mode, infer dry/live from config self.runmode = RunMode.DRY_RUN if config.get('dry_run', True) else RunMode.LIVE - logger.info("Runmode set to {self.runmode}.") + logger.info(f"Runmode set to {self.runmode}.") config.update({'runmode': self.runmode}) - def _validate_config_consistency(self, conf: Dict[str, Any]) -> None: - """ - Validate the configuration consistency - :param conf: Config in JSON format - :return: Returns None if everything is ok, otherwise throw an OperationalException - """ - # validating trailing stoploss - self._validate_trailing_stoploss(conf) - - def _validate_trailing_stoploss(self, conf: Dict[str, Any]) -> None: - - # Skip if trailing stoploss is not activated - if not conf.get('trailing_stop', False): - return - - tsl_positive = float(conf.get('trailing_stop_positive', 0)) - tsl_offset = float(conf.get('trailing_stop_positive_offset', 0)) - tsl_only_offset = conf.get('trailing_only_offset_is_reached', False) - - if tsl_only_offset: - if tsl_positive == 0.0: - raise OperationalException( - f'The config trailing_only_offset_is_reached needs ' - 'trailing_stop_positive_offset to be more than 0 in your config.') - if tsl_positive > 0 and 0 < tsl_offset <= tsl_positive: - raise OperationalException( - f'The config trailing_stop_positive_offset needs ' - 'to be greater than trailing_stop_positive_offset in your config.') - def _args_to_config(self, config: Dict[str, Any], argname: str, logstring: str, logfun: Optional[Callable] = None, deprecated_msg: Optional[str] = None) -> None: @@ -332,3 +326,38 @@ class Configuration(object): logger.info(logstring.format(config[argname])) if deprecated_msg: warnings.warn(f"DEPRECATED: {deprecated_msg}", DeprecationWarning) + + def _resolve_pairs_list(self, config: Dict[str, Any]) -> None: + """ + Helper for download script. + Takes first found: + * -p (pairs argument) + * --pairs-file + * whitelist from config + """ + + if "pairs" in config: + return + + if "pairs_file" in self.args and self.args.pairs_file: + pairs_file = Path(self.args.pairs_file) + logger.info(f'Reading pairs file "{pairs_file}".') + # Download pairs from the pairs file if no config is specified + # or if pairs file is specified explicitely + if not pairs_file.exists(): + raise OperationalException(f'No pairs file found with path "{pairs_file}".') + + config['pairs'] = json_load(pairs_file) + + config['pairs'].sort() + return + + if "config" in self.args and self.args.config: + logger.info("Using pairlist from configuration.") + config['pairs'] = config.get('exchange', {}).get('pair_whitelist') + else: + # Fall back to /dl_path/pairs.json + pairs_file = Path(config['datadir']) / "pairs.json" + if pairs_file.exists(): + config['pairs'] = json_load(pairs_file) + config['pairs'].sort() diff --git a/freqtrade/configuration/json_schema.py b/freqtrade/configuration/json_schema.py deleted file mode 100644 index 4c6f4a4a0..000000000 --- a/freqtrade/configuration/json_schema.py +++ /dev/null @@ -1,53 +0,0 @@ -import logging -from typing import Any, Dict - -from jsonschema import Draft4Validator, validators -from jsonschema.exceptions import ValidationError, best_match - -from freqtrade import constants - - -logger = logging.getLogger(__name__) - - -def _extend_validator(validator_class): - """ - Extended validator for the Freqtrade configuration JSON Schema. - Currently it only handles defaults for subschemas. - """ - validate_properties = validator_class.VALIDATORS['properties'] - - def set_defaults(validator, properties, instance, schema): - for prop, subschema in properties.items(): - if 'default' in subschema: - instance.setdefault(prop, subschema['default']) - - for error in validate_properties( - validator, properties, instance, schema, - ): - yield error - - return validators.extend( - validator_class, {'properties': set_defaults} - ) - - -FreqtradeValidator = _extend_validator(Draft4Validator) - - -def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: - """ - Validate the configuration follow the Config Schema - :param conf: Config in JSON format - :return: Returns the config if valid, otherwise throw an exception - """ - try: - FreqtradeValidator(constants.CONF_SCHEMA).validate(conf) - return conf - except ValidationError as e: - logger.critical( - f"Invalid configuration. See config.json.example. Reason: {e}" - ) - raise ValidationError( - best_match(Draft4Validator(constants.CONF_SCHEMA).iter_errors(conf)).message - ) diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index 25504144f..7a3ca1798 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -1,7 +1,7 @@ """ This module contain functions to load the configuration file """ -import json +import rapidjson import logging import sys from typing import Any, Dict @@ -12,6 +12,9 @@ from freqtrade import OperationalException logger = logging.getLogger(__name__) +CONFIG_PARSE_MODE = rapidjson.PM_COMMENTS | rapidjson.PM_TRAILING_COMMAS + + def load_config_file(path: str) -> Dict[str, Any]: """ Loads a config file from the given path @@ -21,7 +24,7 @@ def load_config_file(path: str) -> Dict[str, Any]: try: # Read config from stdin if requested in the options with open(path) if path != '-' else sys.stdin as file: - config = json.load(file) + config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE) except FileNotFoundError: raise OperationalException( f'Config file "{path}" not found!' diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py new file mode 100644 index 000000000..f980b71ea --- /dev/null +++ b/freqtrade/configuration/timerange.py @@ -0,0 +1,70 @@ +""" +This module contains the argument manager class +""" +import re +from typing import Optional + +import arrow + + +class TimeRange(): + """ + object defining timerange inputs. + [start/stop]type defines if [start/stop]ts shall be used. + if *type is None, don't use corresponding startvalue. + """ + + def __init__(self, starttype: Optional[str] = None, stoptype: Optional[str] = None, + startts: int = 0, stopts: int = 0): + + self.starttype: Optional[str] = starttype + self.stoptype: Optional[str] = stoptype + self.startts: int = startts + self.stopts: int = stopts + + def __eq__(self, other): + """Override the default Equals behavior""" + return (self.starttype == other.starttype and self.stoptype == other.stoptype + and self.startts == other.startts and self.stopts == other.stopts) + + @staticmethod + def parse_timerange(text: Optional[str]): + """ + Parse the value of the argument --timerange to determine what is the range desired + :param text: value from --timerange + :return: Start and End range period + """ + if text is None: + return TimeRange(None, None, 0, 0) + syntax = [(r'^-(\d{8})$', (None, 'date')), + (r'^(\d{8})-$', ('date', None)), + (r'^(\d{8})-(\d{8})$', ('date', 'date')), + (r'^-(\d{10})$', (None, 'date')), + (r'^(\d{10})-$', ('date', None)), + (r'^(\d{10})-(\d{10})$', ('date', 'date')), + (r'^(-\d+)$', (None, 'line')), + (r'^(\d+)-$', ('line', None)), + (r'^(\d+)-(\d+)$', ('index', 'index'))] + for rex, stype in syntax: + # Apply the regular expression to text + match = re.match(rex, text) + if match: # Regex has matched + rvals = match.groups() + index = 0 + start: int = 0 + stop: int = 0 + if stype[0]: + starts = rvals[index] + if stype[0] == 'date' and len(starts) == 8: + start = arrow.get(starts, 'YYYYMMDD').timestamp + else: + start = int(starts) + index += 1 + if stype[1]: + stops = rvals[index] + if stype[1] == 'date' and len(stops) == 8: + stop = arrow.get(stops, 'YYYYMMDD').timestamp + else: + stop = int(stops) + return TimeRange(stype[0], stype[1], start, stop) + raise Exception('Incorrect syntax for timerange "%s"' % text) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 9b73adcfe..05ee99c1b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -5,7 +5,6 @@ bot constants """ DEFAULT_CONFIG = 'config.json' DEFAULT_EXCHANGE = 'bittrex' -DYNAMIC_WHITELIST = 20 # pairs PROCESS_THROTTLE_SECS = 5 # sec DEFAULT_TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs @@ -23,7 +22,6 @@ ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList'] DRY_RUN_WALLET = 999.9 -DEFAULT_DOWNLOAD_TICKER_INTERVALS = '1m 5m' TICKER_INTERVALS = [ '1m', '3m', '5m', '15m', '30m', @@ -39,6 +37,20 @@ SUPPORTED_FIAT = [ "BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT" ] +MINIMAL_CONFIG = { + 'stake_currency': '', + 'dry_run': True, + 'exchange': { + 'name': '', + 'key': '', + 'secret': '', + 'pair_whitelist': [], + 'ccxt_async_config': { + 'enableRateLimit': True, + } + } +} + # Required json-schema for user specified config CONF_SCHEMA = { 'type': 'object', diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index b87589df7..5b71c21a8 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -44,36 +44,49 @@ class DataProvider(): def ohlcv(self, pair: str, ticker_interval: str = None, copy: bool = True) -> DataFrame: """ - get ohlcv data for the given pair as DataFrame - Please check `available_pairs` to verify which pairs are currently cached. + Get ohlcv data for the given pair as DataFrame + Please use the `available_pairs` method to verify which pairs are currently cached. :param pair: pair to get the data for - :param ticker_interval: ticker_interval to get pair for - :param copy: copy dataframe before returning. - Use false only for RO operations (where the dataframe is not modified) + :param ticker_interval: ticker interval to get data for + :param copy: copy dataframe before returning if True. + Use False only for read-only operations (where the dataframe is not modified) """ if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - if ticker_interval: - pairtick = (pair, ticker_interval) - else: - pairtick = (pair, self._config['ticker_interval']) - - return self._exchange.klines(pairtick, copy=copy) + return self._exchange.klines((pair, ticker_interval or self._config['ticker_interval']), + copy=copy) else: return DataFrame() - def historic_ohlcv(self, pair: str, ticker_interval: str) -> DataFrame: + def historic_ohlcv(self, pair: str, ticker_interval: str = None) -> DataFrame: """ - get stored historic ohlcv data + Get stored historic ohlcv data :param pair: pair to get the data for - :param ticker_interval: ticker_interval to get pair for + :param ticker_interval: ticker interval to get data for """ return load_pair_history(pair=pair, - ticker_interval=ticker_interval, + ticker_interval=ticker_interval or self._config['ticker_interval'], refresh_pairs=False, datadir=Path(self._config['datadir']) if self._config.get( 'datadir') else None ) + def get_pair_dataframe(self, pair: str, ticker_interval: str = None) -> DataFrame: + """ + Return pair ohlcv data, either live or cached historical -- depending + on the runmode. + :param pair: pair to get the data for + :param ticker_interval: ticker interval to get data for + """ + if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): + # Get live ohlcv data. + data = self.ohlcv(pair=pair, ticker_interval=ticker_interval) + else: + # Get historic ohlcv data (cached on disk). + data = self.historic_ohlcv(pair=pair, ticker_interval=ticker_interval) + if len(data) == 0: + logger.warning(f"No data found for ({pair}, {ticker_interval}).") + return data + def ticker(self, pair: str): """ Return last ticker data diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index f600615df..007357d9a 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -43,7 +43,7 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: start_index += 1 if timerange.stoptype == 'line': - start_index = len(tickerlist) + timerange.stopts + start_index = max(len(tickerlist) + timerange.stopts, 0) if timerange.stoptype == 'index': stop_index = timerange.stopts elif timerange.stoptype == 'date': @@ -57,10 +57,8 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: return tickerlist[start_index:stop_index] -def load_tickerdata_file( - datadir: Optional[Path], pair: str, - ticker_interval: str, - timerange: Optional[TimeRange] = None) -> Optional[list]: +def load_tickerdata_file(datadir: Optional[Path], pair: str, ticker_interval: str, + timerange: Optional[TimeRange] = None) -> Optional[list]: """ Load a pair from file, either .json.gz or .json :return: tickerlist or None if unsuccesful @@ -68,13 +66,22 @@ def load_tickerdata_file( filename = pair_data_filename(datadir, pair, ticker_interval) pairdata = misc.file_load_json(filename) if not pairdata: - return None + return [] if timerange: pairdata = trim_tickerlist(pairdata, timerange) return pairdata +def store_tickerdata_file(datadir: Optional[Path], pair: str, + ticker_interval: str, data: list, is_zip: bool = False): + """ + Stores tickerdata to file + """ + filename = pair_data_filename(datadir, pair, ticker_interval) + misc.file_dump_json(filename, data, is_zip=is_zip) + + def load_pair_history(pair: str, ticker_interval: str, datadir: Optional[Path], @@ -122,7 +129,7 @@ def load_pair_history(pair: str, else: logger.warning( f'No history data for pair: "{pair}", interval: {ticker_interval}. ' - 'Use --refresh-pairs-cached option or download_backtest_data.py ' + 'Use --refresh-pairs-cached option or `freqtrade download-data` ' 'script to download the data' ) return None @@ -177,11 +184,14 @@ def pair_data_filename(datadir: Optional[Path], pair: str, ticker_interval: str) return filename -def load_cached_data_for_updating(filename: Path, ticker_interval: str, +def load_cached_data_for_updating(datadir: Optional[Path], pair: str, ticker_interval: str, timerange: Optional[TimeRange]) -> Tuple[List[Any], Optional[int]]: """ - Load cached data and choose what part of the data should be updated + Load cached data to download more data. + If timerange is passed in, checks wether data from an before the stored data will be downloaded. + If that's the case than what's available should be completely overwritten. + Only used by download_pair_history(). """ since_ms = None @@ -195,12 +205,11 @@ def load_cached_data_for_updating(filename: Path, ticker_interval: str, since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 # read the cached file - if filename.is_file(): - with open(filename, "rt") as file: - data = misc.json_load(file) - # remove the last item, could be incomplete candle - if data: - data.pop() + # Intentionally don't pass timerange in - since we need to load the full dataset. + data = load_tickerdata_file(datadir, pair, ticker_interval) + # remove the last item, could be incomplete candle + if data: + data.pop() else: data = [] @@ -239,29 +248,28 @@ def download_pair_history(datadir: Optional[Path], ) try: - filename = pair_data_filename(datadir, pair, ticker_interval) - logger.info( f'Download history data for pair: "{pair}", interval: {ticker_interval} ' f'and store in {datadir}.' ) - data, since_ms = load_cached_data_for_updating(filename, ticker_interval, timerange) + data, since_ms = load_cached_data_for_updating(datadir, pair, ticker_interval, timerange) logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') # Default since_ms to 30 days if nothing is given - new_data = exchange.get_history(pair=pair, ticker_interval=ticker_interval, - since_ms=since_ms if since_ms - else - int(arrow.utcnow().shift(days=-30).float_timestamp) * 1000) + new_data = exchange.get_historic_ohlcv(pair=pair, ticker_interval=ticker_interval, + since_ms=since_ms if since_ms + else + int(arrow.utcnow().shift( + days=-30).float_timestamp) * 1000) data.extend(new_data) logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) - misc.file_dump_json(filename, data) + store_tickerdata_file(datadir, pair, ticker_interval, data=data) return True except Exception as e: diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 7085663d6..2d3097ec4 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -10,7 +10,7 @@ import utils_find_1st as utf1st from pandas import DataFrame from freqtrade import constants, OperationalException -from freqtrade.configuration import Arguments, TimeRange +from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.strategy.interface import SellType @@ -75,7 +75,7 @@ class Edge(): self._stoploss_range_step ) - self._timerange: TimeRange = Arguments.parse_timerange("%s-" % arrow.now().shift( + self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift( days=-1 * self._since_number_of_days).format('YYYYMMDD')) self.fee = self.exchange.get_fee() diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index cbf851fc2..4037ca704 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -6,6 +6,8 @@ from freqtrade.exchange.exchange import (get_exchange_bad_reason, # noqa: F401 available_exchanges) from freqtrade.exchange.exchange import (timeframe_to_seconds, # noqa: F401 timeframe_to_minutes, - timeframe_to_msecs) + timeframe_to_msecs, + timeframe_to_next_date, + timeframe_to_prev_date) from freqtrade.exchange.kraken import Kraken # noqa: F401 from freqtrade.exchange.binance import Binance # noqa: F401 diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 0e23cf8b8..7aafff96a 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -6,7 +6,7 @@ import asyncio import inspect import logging from copy import deepcopy -from datetime import datetime +from datetime import datetime, timezone from math import ceil, floor from random import randint from typing import Any, Dict, List, Optional, Tuple @@ -376,7 +376,7 @@ class Exchange(object): 'side': side, 'remaining': amount, 'datetime': arrow.utcnow().isoformat(), - 'status': "open", + 'status': "closed" if ordertype == "market" else "open", 'fee': None, "info": {} } @@ -408,12 +408,12 @@ class Exchange(object): except ccxt.InsufficientFunds as e: raise DependencyException( f'Insufficient funds to create {ordertype} {side} order on market {pair}.' - f'Tried to {side} amount {amount} at rate {rate} (total {rate * amount}).' + f'Tried to {side} amount {amount} at rate {rate}.' f'Message: {e}') from e except ccxt.InvalidOrder as e: raise DependencyException( f'Could not create {ordertype} {side} order on market {pair}.' - f'Tried to {side} amount {amount} at rate {rate} (total {rate * amount}).' + f'Tried to {side} amount {amount} at rate {rate}.' f'Message: {e}') from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( @@ -472,7 +472,7 @@ class Exchange(object): order = self.create_order(pair, ordertype, 'sell', amount, rate, params) logger.info('stoploss limit order added for %s. ' - 'stop price: %s. limit: %s' % (pair, stop_price, rate)) + 'stop price: %s. limit: %s', pair, stop_price, rate) return order @retrier @@ -546,19 +546,24 @@ class Exchange(object): logger.info("returning cached ticker-data for %s", pair) return self._cached_ticker[pair] - def get_history(self, pair: str, ticker_interval: str, - since_ms: int) -> List: + def get_historic_ohlcv(self, pair: str, ticker_interval: str, + since_ms: int) -> List: """ Gets candle history using asyncio and returns the list of candles. Handles all async doing. + Async over one pair, assuming we get `_ohlcv_candle_limit` candles per call. + :param pair: Pair to download + :param ticker_interval: Interval to get + :param since_ms: Timestamp in milliseconds to get history from + :returns List of tickers """ return asyncio.get_event_loop().run_until_complete( - self._async_get_history(pair=pair, ticker_interval=ticker_interval, - since_ms=since_ms)) + self._async_get_historic_ohlcv(pair=pair, ticker_interval=ticker_interval, + since_ms=since_ms)) - async def _async_get_history(self, pair: str, - ticker_interval: str, - since_ms: int) -> List: + async def _async_get_historic_ohlcv(self, pair: str, + ticker_interval: str, + since_ms: int) -> List: one_call = timeframe_to_msecs(ticker_interval) * self._ohlcv_candle_limit logger.debug( @@ -584,7 +589,10 @@ class Exchange(object): def refresh_latest_ohlcv(self, pair_list: List[Tuple[str, str]]) -> List[Tuple[str, List]]: """ - Refresh in-memory ohlcv asyncronously and set `_klines` with the result + Refresh in-memory ohlcv asynchronously and set `_klines` with the result + Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). + :param pair_list: List of 2 element tuples containing pair, interval to refresh + :return: Returns a List of ticker-dataframes. """ logger.debug("Refreshing ohlcv data for %d pairs", len(pair_list)) @@ -632,7 +640,7 @@ class Exchange(object): async def _async_get_candle_history(self, pair: str, ticker_interval: str, since_ms: Optional[int] = None) -> Tuple[str, str, List]: """ - Asyncronously gets candle histories using fetch_ohlcv + Asynchronously gets candle histories using fetch_ohlcv returns tuple: (pair, ticker_interval, ohlcv_list) """ try: @@ -688,8 +696,13 @@ class Exchange(object): @retrier def get_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - order = self._dry_run_open_orders[order_id] - return order + try: + order = self._dry_run_open_orders[order_id] + return order + except KeyError as e: + # Gracefully handle errors with dry-run orders. + raise InvalidOrderException( + f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e try: return self._api.fetch_order(order_id, pair) except ccxt.InvalidOrder as e: @@ -790,13 +803,45 @@ def timeframe_to_seconds(ticker_interval: str) -> int: def timeframe_to_minutes(ticker_interval: str) -> int: """ - Same as above, but returns minutes. + Same as timeframe_to_seconds, but returns minutes. """ return ccxt.Exchange.parse_timeframe(ticker_interval) // 60 def timeframe_to_msecs(ticker_interval: str) -> int: """ - Same as above, but returns milliseconds. + Same as timeframe_to_seconds, but returns milliseconds. """ return ccxt.Exchange.parse_timeframe(ticker_interval) * 1000 + + +def timeframe_to_prev_date(timeframe: str, date: datetime = None) -> datetime: + """ + Use Timeframe and determine last possible candle. + :param timeframe: timeframe in string format (e.g. "5m") + :param date: date to use. Defaults to utcnow() + :returns: date of previous candle (with utc timezone) + """ + if not date: + date = datetime.now(timezone.utc) + timeframe_secs = timeframe_to_seconds(timeframe) + # Get offset based on timerame_secs + offset = date.timestamp() % timeframe_secs + # Subtract seconds passed since last offset + new_timestamp = date.timestamp() - offset + return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) + + +def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: + """ + Use Timeframe and determine next candle. + :param timeframe: timeframe in string format (e.g. "5m") + :param date: date to use. Defaults to utcnow() + :returns: date of next candle (with utc timezone) + """ + prevdate = timeframe_to_prev_date(timeframe, date) + timeframe_secs = timeframe_to_seconds(timeframe) + + # Add one interval to previous candle + new_timestamp = prevdate.timestamp() + timeframe_secs + return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 603b0631f..e5ecef8bf 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -16,7 +16,8 @@ from freqtrade import (DependencyException, OperationalException, InvalidOrderEx from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.exchange import timeframe_to_minutes +from freqtrade.configuration import validate_config_consistency +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.resolvers import ExchangeResolver, StrategyResolver, PairListResolver @@ -51,6 +52,9 @@ class FreqtradeBot(object): self.strategy: IStrategy = StrategyResolver(self.config).strategy + # Check config consistency here since strategies can set certain options + validate_config_consistency(config) + self.rpc: RPCManager = RPCManager(self) self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange @@ -105,13 +109,12 @@ class FreqtradeBot(object): # Adjust stoploss if it was changed Trade.stoploss_reinitialization(self.strategy.stoploss) - def process(self) -> bool: + def process(self) -> None: """ Queries the persistence layer for open trades and handles them, otherwise a new trade is created. :return: True if one or more trades has been created or closed, False otherwise """ - state_changed = False # Check whether markets have to be reloaded self.exchange._reload_markets() @@ -138,19 +141,17 @@ class FreqtradeBot(object): # First process current opened trades for trade in trades: - state_changed |= self.process_maybe_execute_sell(trade) + self.process_maybe_execute_sell(trade) # Then looking for buy opportunities if len(trades) < self.config['max_open_trades']: - state_changed = self.process_maybe_execute_buy() + self.process_maybe_execute_buy() if 'unfilledtimeout' in self.config: # Check and handle any timed out open orders self.check_handle_timedout() Trade.session.flush() - return state_changed - def _extend_whitelist_with_trades(self, whitelist: List[str], trades: List[Any]): """ Extend whitelist with pairs from open trades @@ -259,11 +260,12 @@ class FreqtradeBot(object): amount_reserve_percent = max(amount_reserve_percent, 0.5) return min(min_stake_amounts) / amount_reserve_percent - def create_trade(self) -> bool: + def create_trades(self) -> bool: """ - Checks the implemented trading indicator(s) for a randomly picked pair, - if one pair triggers the buy_signal a new trade record gets created - :return: True if a trade object has been created and persisted, False otherwise + Checks the implemented trading strategy for buy-signals, using the active pair whitelist. + If a pair triggers the buy_signal a new trade record gets created. + Checks pairs as long as the open trade count is below `max_open_trades`. + :return: True if at least one trade has been created. """ interval = self.strategy.ticker_interval whitelist = copy.deepcopy(self.active_pair_whitelist) @@ -282,15 +284,19 @@ class FreqtradeBot(object): logger.info("No currency pair in whitelist, but checking to sell open trades.") return False + buycount = 0 # running get_signal on historical data fetched for _pair in whitelist: + if self.strategy.is_pair_locked(_pair): + logger.info(f"Pair {_pair} is currently locked.") + continue (buy, sell) = self.strategy.get_signal( _pair, interval, self.dataprovider.ohlcv(_pair, self.strategy.ticker_interval)) - if buy and not sell: + if buy and not sell and len(Trade.get_open_trades()) < self.config['max_open_trades']: stake_amount = self._get_trade_stake_amount(_pair) if not stake_amount: - return False + continue logger.info(f"Buy signal found: about create a new trade with stake_amount: " f"{stake_amount} ...") @@ -300,12 +306,13 @@ class FreqtradeBot(object): if (bidstrat_check_depth_of_market.get('enabled', False)) and\ (bidstrat_check_depth_of_market.get('bids_to_ask_delta', 0) > 0): if self._check_depth_of_market_buy(_pair, bidstrat_check_depth_of_market): - return self.execute_buy(_pair, stake_amount) + buycount += self.execute_buy(_pair, stake_amount) else: - return False - return self.execute_buy(_pair, stake_amount) + continue - return False + buycount += self.execute_buy(_pair, stake_amount) + + return buycount > 0 def _check_depth_of_market_buy(self, pair: str, conf: Dict) -> bool: """ @@ -429,21 +436,17 @@ class FreqtradeBot(object): return True - def process_maybe_execute_buy(self) -> bool: + def process_maybe_execute_buy(self) -> None: """ Tries to execute a buy trade in a safe way :return: True if executed """ try: # Create entity and execute trade - if self.create_trade(): - return True - - logger.info('Found no buy signals for whitelisted currencies. Trying again..') - return False + if not self.create_trades(): + logger.info('Found no buy signals for whitelisted currencies. Trying again...') except DependencyException as exception: logger.warning('Unable to create trade: %s', exception) - return False def process_maybe_execute_sell(self, trade: Trade) -> bool: """ @@ -678,6 +681,9 @@ class FreqtradeBot(object): if stoploss_order and stoploss_order['status'] == 'closed': trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value trade.update(stoploss_order) + # Lock pair for one candle to prevent immediate rebuys + self.strategy.lock_pair(trade.pair, + timeframe_to_next_date(self.config['ticker_interval'])) self._notify_sell(trade) return True @@ -875,16 +881,23 @@ class FreqtradeBot(object): logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}") # Execute sell and update trade record - order_id = self.exchange.sell(pair=str(trade.pair), - ordertype=self.strategy.order_types[sell_type], - amount=trade.amount, rate=limit, - time_in_force=self.strategy.order_time_in_force['sell'] - )['id'] + order = self.exchange.sell(pair=str(trade.pair), + ordertype=self.strategy.order_types[sell_type], + amount=trade.amount, rate=limit, + time_in_force=self.strategy.order_time_in_force['sell'] + ) - trade.open_order_id = order_id + trade.open_order_id = order['id'] trade.close_rate_requested = limit trade.sell_reason = sell_reason.value + # In case of market sell orders the order can be closed immediately + if order.get('status', 'unknown') == 'closed': + trade.update(order) Trade.session.flush() + + # Lock pair for one candle to prevent immediate rebuys + self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['ticker_interval'])) + self._notify_sell(trade) def _notify_sell(self, trade: Trade): diff --git a/freqtrade/misc.py b/freqtrade/misc.py index ed9f5c1e9..e68279921 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -5,11 +5,11 @@ import gzip import logging import re from datetime import datetime +from pathlib import Path import numpy as np import rapidjson - logger = logging.getLogger(__name__) @@ -39,7 +39,7 @@ def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray: return dates.dt.to_pydatetime() -def file_dump_json(filename, data, is_zip=False) -> None: +def file_dump_json(filename: Path, data, is_zip=False) -> None: """ Dump JSON data into a file :param filename: file to create @@ -49,8 +49,8 @@ def file_dump_json(filename, data, is_zip=False) -> None: logger.info(f'dumping json to "{filename}"') if is_zip: - if not filename.endswith('.gz'): - filename = filename + '.gz' + if filename.suffix != '.gz': + filename = filename.with_suffix('.gz') with gzip.open(filename, 'w') as fp: rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) else: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 252175269..3878d7603 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -12,7 +12,7 @@ from typing import Any, Dict, List, NamedTuple, Optional from pandas import DataFrame from freqtrade import OperationalException -from freqtrade.configuration import Arguments +from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.dataprovider import DataProvider from freqtrade.exchange import timeframe_to_minutes @@ -190,7 +190,7 @@ class Backtesting(object): return tabulate(tabular_data, headers=headers, # type: ignore floatfmt=floatfmt, tablefmt="pipe") - def _store_backtest_result(self, recordfilename: str, results: DataFrame, + def _store_backtest_result(self, recordfilename: Path, results: DataFrame, strategyname: Optional[str] = None) -> None: records = [(t.pair, t.profit_percent, t.open_time.timestamp(), @@ -201,10 +201,10 @@ class Backtesting(object): if records: if strategyname: # Inject strategyname to filename - recname = Path(recordfilename) - recordfilename = str(Path.joinpath( - recname.parent, f'{recname.stem}-{strategyname}').with_suffix(recname.suffix)) - logger.info('Dumping backtest results to %s', recordfilename) + recordfilename = Path.joinpath( + recordfilename.parent, + f'{recordfilename.stem}-{strategyname}').with_suffix(recordfilename.suffix) + logger.info(f'Dumping backtest results to {recordfilename}') file_dump_json(recordfilename, records) def _get_ticker_list(self, processed) -> Dict[str, DataFrame]: @@ -404,7 +404,7 @@ class Backtesting(object): logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - timerange = Arguments.parse_timerange(None if self.config.get( + timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = history.load_data( datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, @@ -458,7 +458,7 @@ class Backtesting(object): for strategy, results in all_results.items(): if self.config.get('export', False): - self._store_backtest_result(self.config['exportfilename'], results, + self._store_backtest_result(Path(self.config['exportfilename']), results, strategy if len(self.strategylist) > 1 else None) print(f"Result for strategy {strategy}") diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index 8d1fa381b..7e0d60843 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -9,7 +9,7 @@ from tabulate import tabulate from freqtrade import constants from freqtrade.edge import Edge -from freqtrade.configuration import Arguments +from freqtrade.configuration import TimeRange from freqtrade.exchange import Exchange from freqtrade.resolvers import StrategyResolver @@ -41,7 +41,7 @@ class EdgeCli(object): self.edge = Edge(config, self.exchange, self.strategy) self.edge._refresh_pairs = self.config.get('refresh_pairs', False) - self.timerange = Arguments.parse_timerange(None if self.config.get( + self.timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) self.edge._timerange = self.timerange diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index eaa3a7e1d..b474ed318 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -8,11 +8,14 @@ import logging import os import sys +from collections import OrderedDict from operator import itemgetter from pathlib import Path from pprint import pprint from typing import Any, Dict, List, Optional +import rapidjson + from colorama import init as colorama_init from colorama import Fore, Style from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects, cpu_count @@ -20,7 +23,7 @@ from pandas import DataFrame from skopt import Optimizer from skopt.space import Dimension -from freqtrade.configuration import Arguments +from freqtrade.configuration import TimeRange from freqtrade.data.history import load_data, get_timeframe from freqtrade.misc import round_dict from freqtrade.optimize.backtesting import Backtesting @@ -135,24 +138,46 @@ class Hyperopt(Backtesting): results = sorted(self.trials, key=itemgetter('loss')) best_result = results[0] params = best_result['params'] - log_str = self.format_results_logstring(best_result) print(f"\nBest result:\n\n{log_str}\n") - if self.has_space('buy'): - print('Buy hyperspace params:') - pprint({p.name: params.get(p.name) for p in self.hyperopt_space('buy')}, - indent=4) - if self.has_space('sell'): - print('Sell hyperspace params:') - pprint({p.name: params.get(p.name) for p in self.hyperopt_space('sell')}, - indent=4) - if self.has_space('roi'): - print("ROI table:") - # Round printed values to 5 digits after the decimal point - pprint(round_dict(self.custom_hyperopt.generate_roi_table(params), 5), indent=4) - if self.has_space('stoploss'): - # Also round to 5 digits after the decimal point - print(f"Stoploss: {round(params.get('stoploss'), 5)}") + + if self.config.get('print_json'): + result_dict: Dict = {} + if self.has_space('buy') or self.has_space('sell'): + result_dict['params'] = {} + if self.has_space('buy'): + result_dict['params'].update({p.name: params.get(p.name) + for p in self.hyperopt_space('buy')}) + if self.has_space('sell'): + result_dict['params'].update({p.name: params.get(p.name) + for p in self.hyperopt_space('sell')}) + if self.has_space('roi'): + # Convert keys in min_roi dict to strings because + # rapidjson cannot dump dicts with integer keys... + # OrderedDict is used to keep the numeric order of the items + # in the dict. + result_dict['minimal_roi'] = OrderedDict( + (str(k), v) for k, v in self.custom_hyperopt.generate_roi_table(params).items() + ) + if self.has_space('stoploss'): + result_dict['stoploss'] = params.get('stoploss') + print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE)) + else: + if self.has_space('buy'): + print('Buy hyperspace params:') + pprint({p.name: params.get(p.name) for p in self.hyperopt_space('buy')}, + indent=4) + if self.has_space('sell'): + print('Sell hyperspace params:') + pprint({p.name: params.get(p.name) for p in self.hyperopt_space('sell')}, + indent=4) + if self.has_space('roi'): + print("ROI table:") + # Round printed values to 5 digits after the decimal point + pprint(round_dict(self.custom_hyperopt.generate_roi_table(params), 5), indent=4) + if self.has_space('stoploss'): + # Also round to 5 digits after the decimal point + print(f"Stoploss: {round(params.get('stoploss'), 5)}") def log_results(self, results) -> None: """ @@ -314,7 +339,7 @@ class Hyperopt(Backtesting): ) def start(self) -> None: - timerange = Arguments.parse_timerange(None if self.config.get( + timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = load_data( datadir=Path(self.config['datadir']) if self.config.get('datadir') else None, diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 9a2e2eac4..b9b7977ab 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -55,7 +55,6 @@ class VolumePairList(IPairList): # Generate dynamic whitelist self._whitelist = self._gen_pair_whitelist( self._config['stake_currency'], self._sort_key)[:self._number_pairs] - logger.info(f"Searching pairs: {self._whitelist}") @cached(TTLCache(maxsize=1, ttl=1800)) def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: @@ -92,4 +91,6 @@ class VolumePairList(IPairList): valid_tickers.remove(t) pairs = [s['symbol'] for s in valid_tickers] + logger.info(f"Searching pairs: {self._whitelist}") + return pairs diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index d03d3ae53..e6da581a4 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -4,7 +4,7 @@ from typing import Dict, List, Optional import pandas as pd -from freqtrade.configuration import Arguments +from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import (combine_tickers_with_mean, create_cum_profit, load_trades) @@ -37,12 +37,12 @@ def init_plotscript(config): strategy = StrategyResolver(config).strategy if "pairs" in config: - pairs = config["pairs"].split(',') + pairs = config["pairs"] else: pairs = config["exchange"]["pair_whitelist"] # Set timerange to use - timerange = Arguments.parse_timerange(config.get("timerange")) + timerange = TimeRange.parse_timerange(config.get("timerange")) tickers = history.load_data( datadir=Path(str(config.get("datadir"))), diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 841c3cf43..310c54015 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -29,7 +29,8 @@ class IResolver(object): """ # Generate spec based on absolute path - spec = importlib.util.spec_from_file_location('unknown', str(module_path)) + # Pass object_name as first argument to have logging print a reasonable name. + spec = importlib.util.spec_from_file_location(object_name, str(module_path)) module = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(module) # type: ignore # importlib does not use typehints diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 37aa97bb1..99f5f26de 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -4,7 +4,7 @@ This module defines the interface to apply for strategies """ import logging from abc import ABC, abstractmethod -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from typing import Dict, List, NamedTuple, Optional, Tuple import warnings @@ -107,6 +107,7 @@ class IStrategy(ABC): self.config = config # Dict to determine if analysis is necessary self._last_candle_seen_per_pair: Dict[str, datetime] = {} + self._pair_locked_until: Dict[str, datetime] = {} @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -154,6 +155,24 @@ class IStrategy(ABC): """ return self.__class__.__name__ + def lock_pair(self, pair: str, until: datetime) -> None: + """ + Locks pair until a given timestamp happens. + Locked pairs are not analyzed, and are prevented from opening new trades. + :param pair: Pair to lock + :param until: datetime in UTC until the pair should be blocked from opening new trades. + Needs to be timezone aware `datetime.now(timezone.utc)` + """ + self._pair_locked_until[pair] = until + + def is_pair_locked(self, pair: str) -> bool: + """ + Checks if a pair is currently locked + """ + if pair not in self._pair_locked_until: + return False + return self._pair_locked_until[pair] >= datetime.now(timezone.utc) + def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Parses the given ticker history and returns a populated DataFrame @@ -260,8 +279,8 @@ class IStrategy(ABC): sell: bool, low: float = None, high: float = None, force_stoploss: float = 0) -> SellCheckTuple: """ - This function evaluate if on the condition required to trigger a sell has been reached - if the threshold is reached and updates the trade record. + This function evaluates if one of the conditions required to trigger a sell + has been reached, which can either be a stop-loss, ROI or sell-signal. :param low: Only used during backtesting to simulate stoploss :param high: Only used during backtesting, to simulate ROI :param force_stoploss: Externally provided stoploss diff --git a/freqtrade/tests/config_test_comments.json b/freqtrade/tests/config_test_comments.json new file mode 100644 index 000000000..85becc3e8 --- /dev/null +++ b/freqtrade/tests/config_test_comments.json @@ -0,0 +1,133 @@ +{ + /* Single-line C-style comment */ + "max_open_trades": 3, + /* + * Multi-line C-style comment + */ + "stake_currency": "BTC", + "stake_amount": 0.05, + "fiat_display_currency": "USD", // C++-style comment + "amount_reserve_percent" : 0.05, // And more, tabs before this comment + "dry_run": false, + "ticker_interval": "5m", + "trailing_stop": false, + "trailing_stop_positive": 0.005, + "trailing_stop_positive_offset": 0.0051, + "trailing_only_offset_is_reached": false, + "minimal_roi": { + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 + }, + "stoploss": -0.10, + "unfilledtimeout": { + "buy": 10, + "sell": 30, // Trailing comma should also be accepted now + }, + "bid_strategy": { + "use_order_book": false, + "ask_last_balance": 0.0, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 + }, + "order_types": { + "buy": "limit", + "sell": "limit", + "stoploss": "market", + "stoploss_on_exchange": false, + "stoploss_on_exchange_interval": 60 + }, + "order_time_in_force": { + "buy": "gtc", + "sell": "gtc" + }, + "pairlist": { + "method": "VolumePairList", + "config": { + "number_assets": 20, + "sort_key": "quoteVolume", + "precision_filter": false + } + }, + "exchange": { + "name": "bittrex", + "sandbox": false, + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "password": "", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": false, + "rateLimit": 500, + "aiohttp_trust_env": false + }, + "pair_whitelist": [ + "ETH/BTC", + "LTC/BTC", + "ETC/BTC", + "DASH/BTC", + "ZEC/BTC", + "XLM/BTC", + "NXT/BTC", + "POWR/BTC", + "ADA/BTC", + "XMR/BTC" + ], + "pair_blacklist": [ + "DOGE/BTC" + ], + "outdated_offset": 5, + "markets_refresh_interval": 60 + }, + "edge": { + "enabled": false, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 7, + "capital_available_percentage": 0.5, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "minimum_winrate": 0.60, + "minimum_expectancy": 0.20, + "min_trade_number": 10, + "max_trade_duration_minute": 1440, + "remove_pumps": false + }, + "experimental": { + "use_sell_signal": false, + "sell_profit_only": false, + "ignore_roi_if_buy_signal": false + }, + "telegram": { +// We can now comment out some settings +// "enabled": true, + "enabled": false, + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id" + }, + "api_server": { + "enabled": false, + "listen_ip_address": "127.0.0.1", + "listen_port": 8080, + "username": "freqtrader", + "password": "SuperSecurePassword" + }, + "db_url": "sqlite:///tradesv3.sqlite", + "initial_state": "running", + "forcebuy_enable": false, + "internals": { + "process_throttle_secs": 5 + }, + "strategy": "DefaultStrategy", + "strategy_path": "user_data/strategies/" +} diff --git a/freqtrade/tests/data/test_btanalysis.py b/freqtrade/tests/data/test_btanalysis.py index bad8db66f..cf8cae566 100644 --- a/freqtrade/tests/data/test_btanalysis.py +++ b/freqtrade/tests/data/test_btanalysis.py @@ -4,7 +4,7 @@ import pytest from arrow import Arrow from pandas import DataFrame, to_datetime -from freqtrade.configuration import Arguments, TimeRange +from freqtrade.configuration import TimeRange from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, combine_tickers_with_mean, create_cum_profit, @@ -121,7 +121,7 @@ def test_combine_tickers_with_mean(): def test_create_cum_profit(): filename = make_testdata_path(None) / "backtest-result_test.json" bt_data = load_backtest_data(filename) - timerange = Arguments.parse_timerange("20180110-20180112") + timerange = TimeRange.parse_timerange("20180110-20180112") df = load_pair_history(pair="POWR/BTC", ticker_interval='5m', datadir=None, timerange=timerange) diff --git a/freqtrade/tests/data/test_dataprovider.py b/freqtrade/tests/data/test_dataprovider.py index 993f0b59b..2272f69a3 100644 --- a/freqtrade/tests/data/test_dataprovider.py +++ b/freqtrade/tests/data/test_dataprovider.py @@ -13,6 +13,7 @@ def test_ohlcv(mocker, default_conf, ticker_history): exchange = get_patched_exchange(mocker, default_conf) exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history + dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN assert ticker_history.equals(dp.ohlcv("UNITTEST/BTC", ticker_interval)) @@ -37,11 +38,9 @@ def test_ohlcv(mocker, default_conf, ticker_history): def test_historic_ohlcv(mocker, default_conf, ticker_history): - historymock = MagicMock(return_value=ticker_history) mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock) - # exchange = get_patched_exchange(mocker, default_conf) dp = DataProvider(default_conf, None) data = dp.historic_ohlcv("UNITTEST/BTC", "5m") assert isinstance(data, DataFrame) @@ -51,14 +50,47 @@ def test_historic_ohlcv(mocker, default_conf, ticker_history): assert historymock.call_args_list[0][1]["ticker_interval"] == "5m" +def test_get_pair_dataframe(mocker, default_conf, ticker_history): + default_conf["runmode"] = RunMode.DRY_RUN + ticker_interval = default_conf["ticker_interval"] + exchange = get_patched_exchange(mocker, default_conf) + exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history + exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history + + dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.DRY_RUN + assert ticker_history.equals(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)) + assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame) + assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval) is not ticker_history + assert not dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval).empty + assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty + + # Test with and without parameter + assert dp.get_pair_dataframe("UNITTEST/BTC", + ticker_interval).equals(dp.get_pair_dataframe("UNITTEST/BTC")) + + default_conf["runmode"] = RunMode.LIVE + dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.LIVE + assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame) + assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty + + historymock = MagicMock(return_value=ticker_history) + mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock) + default_conf["runmode"] = RunMode.BACKTEST + dp = DataProvider(default_conf, exchange) + assert dp.runmode == RunMode.BACKTEST + assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame) + # assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty + + def test_available_pairs(mocker, default_conf, ticker_history): exchange = get_patched_exchange(mocker, default_conf) - ticker_interval = default_conf["ticker_interval"] exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history - dp = DataProvider(default_conf, exchange) + dp = DataProvider(default_conf, exchange) assert len(dp.available_pairs) == 2 assert dp.available_pairs == [ ("XRP/BTC", ticker_interval), diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 00f4738f7..7360f3c1c 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -74,13 +74,13 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf) -> None: assert ld is None assert log_has( 'No history data for pair: "UNITTEST/BTC", interval: 7m. ' - 'Use --refresh-pairs-cached option or download_backtest_data.py ' + 'Use --refresh-pairs-cached option or `freqtrade download-data` ' 'script to download the data', caplog ) def test_load_data_1min_ticker(ticker_history, mocker, caplog) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history) + mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'UNITTEST_BTC-1m.json') _backup_file(file, copy_file=True) history.load_data(datadir=None, ticker_interval='1m', pairs=['UNITTEST/BTC']) @@ -96,7 +96,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau """ Test load_pair_history() with 1 min ticker """ - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) + mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history_list) exchange = get_patched_exchange(mocker, default_conf) file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') @@ -109,7 +109,7 @@ def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, defau assert os.path.isfile(file) is False assert log_has( 'No history data for pair: "MEME/BTC", interval: 1m. ' - 'Use --refresh-pairs-cached option or download_backtest_data.py ' + 'Use --refresh-pairs-cached option or `freqtrade download-data` ' 'script to download the data', caplog ) @@ -178,16 +178,13 @@ def test_load_cached_data_for_updating(mocker) -> None: # timeframe starts earlier than the cached data # should fully update data timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == [] assert start_ts == test_data[0][0] - 1000 # same with 'line' timeframe num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120 - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', TimeRange(None, 'line', 0, -num_lines)) assert data == [] assert start_ts < test_data[0][0] - 1 @@ -195,36 +192,29 @@ def test_load_cached_data_for_updating(mocker) -> None: # timeframe starts in the center of the cached data # should return the chached data w/o the last item timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] # same with 'line' timeframe num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] # timeframe starts after the chached data # should return the chached data w/o the last item timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] - # same with 'line' timeframe + # Try loading last 30 lines. + # Not supported by load_cached_data_for_updating, we always need to get the full data. num_lines = 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] @@ -232,41 +222,33 @@ def test_load_cached_data_for_updating(mocker) -> None: # should return the chached data w/o the last item num_lines = 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename, - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) assert data == test_data[:-1] assert test_data[-2][0] < start_ts < test_data[-1][0] # no datafile exist # should return timestamp start time timerange = TimeRange('date', None, now_ts - 10000, 0) - data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'), - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange) assert data == [] assert start_ts == (now_ts - 10000) * 1000 # same with 'line' timeframe num_lines = 30 timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'), - '1m', - timerange) + data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange) assert data == [] assert start_ts == (now_ts - num_lines * 60) * 1000 # no datafile exist, no timeframe is set # should return an empty array and None - data, start_ts = load_cached_data_for_updating(test_filename.with_name('unexist'), - '1m', - None) + data, start_ts = load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', None) assert data == [] assert start_ts is None def test_download_pair_history(ticker_history_list, mocker, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=ticker_history_list) + mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history_list) exchange = get_patched_exchange(mocker, default_conf) file1_1 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-1m.json') file1_5 = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'MEME_BTC-5m.json') @@ -319,7 +301,7 @@ def test_download_pair_history2(mocker, default_conf) -> None: [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) - mocker.patch('freqtrade.exchange.Exchange.get_history', return_value=tick) + mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) download_pair_history(None, exchange, pair="UNITTEST/BTC", ticker_interval='1m') download_pair_history(None, exchange, pair="UNITTEST/BTC", ticker_interval='3m') @@ -327,7 +309,7 @@ def test_download_pair_history2(mocker, default_conf) -> None: def test_download_backtesting_data_exception(ticker_history, mocker, caplog, default_conf) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_history', + mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', side_effect=Exception('File Error')) exchange = get_patched_exchange(mocker, default_conf) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index e8a7201f1..e453b5dca 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -14,7 +14,11 @@ from pandas import DataFrame from freqtrade import (DependencyException, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Binance, Exchange, Kraken -from freqtrade.exchange.exchange import API_RETRY_COUNT +from freqtrade.exchange.exchange import (API_RETRY_COUNT, timeframe_to_minutes, + timeframe_to_msecs, + timeframe_to_next_date, + timeframe_to_prev_date, + timeframe_to_seconds) from freqtrade.resolvers.exchange_resolver import ExchangeResolver from freqtrade.tests.conftest import get_patched_exchange, log_has, log_has_re @@ -652,7 +656,13 @@ def test_buy_prod(default_conf, mocker, exchange_name): with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.buy(pair='ETH/BTC', ordertype=order_type, + exchange.buy(pair='ETH/BTC', ordertype='limit', + amount=1, rate=200, time_in_force=time_in_force) + + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + exchange.buy(pair='ETH/BTC', ordertype='market', amount=1, rate=200, time_in_force=time_in_force) with pytest.raises(TemporaryError): @@ -775,7 +785,13 @@ def test_sell_prod(default_conf, mocker, exchange_name): with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + exchange.sell(pair='ETH/BTC', ordertype='limit', amount=1, rate=200) + + # Market orders don't require price, so the behaviour is slightly different + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + exchange.sell(pair='ETH/BTC', ordertype='market', amount=1, rate=200) with pytest.raises(TemporaryError): api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No Connection")) @@ -996,7 +1012,7 @@ def test_get_ticker(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_get_history(default_conf, mocker, caplog, exchange_name): +def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) tick = [ [ @@ -1017,7 +1033,7 @@ def test_get_history(default_conf, mocker, caplog, exchange_name): # one_call calculation * 1.8 should do 2 calls since = 5 * 60 * 500 * 1.8 print(f"since = {since}") - ret = exchange.get_history(pair, "5m", int((arrow.utcnow().timestamp - since) * 1000)) + ret = exchange.get_historic_ohlcv(pair, "5m", int((arrow.utcnow().timestamp - since) * 1000)) assert exchange._async_get_candle_history.call_count == 2 # Returns twice the above tick @@ -1324,6 +1340,9 @@ def test_get_order(default_conf, mocker, exchange_name): print(exchange.get_order('X', 'TKN/BTC')) assert exchange.get_order('X', 'TKN/BTC').myid == 123 + with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'): + exchange.get_order('Y', 'TKN/BTC') + default_conf['dry_run'] = False api_mock = MagicMock() api_mock.fetch_order = MagicMock(return_value=456) @@ -1540,3 +1559,74 @@ def test_get_valid_pair_combination(default_conf, mocker, markets): assert ex.get_valid_pair_combination("BTC", "ETH") == "ETH/BTC" with pytest.raises(DependencyException, match=r"Could not combine.* to get a valid pair."): ex.get_valid_pair_combination("NOPAIR", "ETH") + + +def test_timeframe_to_minutes(): + assert timeframe_to_minutes("5m") == 5 + assert timeframe_to_minutes("10m") == 10 + assert timeframe_to_minutes("1h") == 60 + assert timeframe_to_minutes("1d") == 1440 + + +def test_timeframe_to_seconds(): + assert timeframe_to_seconds("5m") == 300 + assert timeframe_to_seconds("10m") == 600 + assert timeframe_to_seconds("1h") == 3600 + assert timeframe_to_seconds("1d") == 86400 + + +def test_timeframe_to_msecs(): + assert timeframe_to_msecs("5m") == 300000 + assert timeframe_to_msecs("10m") == 600000 + assert timeframe_to_msecs("1h") == 3600000 + assert timeframe_to_msecs("1d") == 86400000 + + +def test_timeframe_to_prev_date(): + # 2019-08-12 13:22:08 + date = datetime.fromtimestamp(1565616128, tz=timezone.utc) + + tf_list = [ + # 5m -> 2019-08-12 13:20:00 + ("5m", datetime(2019, 8, 12, 13, 20, 0, tzinfo=timezone.utc)), + # 10m -> 2019-08-12 13:20:00 + ("10m", datetime(2019, 8, 12, 13, 20, 0, tzinfo=timezone.utc)), + # 1h -> 2019-08-12 13:00:00 + ("1h", datetime(2019, 8, 12, 13, 00, 0, tzinfo=timezone.utc)), + # 2h -> 2019-08-12 12:00:00 + ("2h", datetime(2019, 8, 12, 12, 00, 0, tzinfo=timezone.utc)), + # 4h -> 2019-08-12 12:00:00 + ("4h", datetime(2019, 8, 12, 12, 00, 0, tzinfo=timezone.utc)), + # 1d -> 2019-08-12 00:00:00 + ("1d", datetime(2019, 8, 12, 00, 00, 0, tzinfo=timezone.utc)), + ] + for interval, result in tf_list: + assert timeframe_to_prev_date(interval, date) == result + + date = datetime.now(tz=timezone.utc) + assert timeframe_to_prev_date("5m", date) < date + + +def test_timeframe_to_next_date(): + # 2019-08-12 13:22:08 + date = datetime.fromtimestamp(1565616128, tz=timezone.utc) + tf_list = [ + # 5m -> 2019-08-12 13:25:00 + ("5m", datetime(2019, 8, 12, 13, 25, 0, tzinfo=timezone.utc)), + # 10m -> 2019-08-12 13:30:00 + ("10m", datetime(2019, 8, 12, 13, 30, 0, tzinfo=timezone.utc)), + # 1h -> 2019-08-12 14:00:00 + ("1h", datetime(2019, 8, 12, 14, 00, 0, tzinfo=timezone.utc)), + # 2h -> 2019-08-12 14:00:00 + ("2h", datetime(2019, 8, 12, 14, 00, 0, tzinfo=timezone.utc)), + # 4h -> 2019-08-12 14:00:00 + ("4h", datetime(2019, 8, 12, 16, 00, 0, tzinfo=timezone.utc)), + # 1d -> 2019-08-13 00:00:00 + ("1d", datetime(2019, 8, 13, 0, 0, 0, tzinfo=timezone.utc)), + ] + + for interval, result in tf_list: + assert timeframe_to_next_date(interval, date) == result + + date = datetime.now(tz=timezone.utc) + assert timeframe_to_next_date("5m", date) > date diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 9ed7e7296..02e9a9c28 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -2,6 +2,7 @@ import math import random +from pathlib import Path from unittest.mock import MagicMock import numpy as np @@ -785,10 +786,10 @@ def test_backtest_record(default_conf, fee, mocker): # reset test to test with strategy name names = [] records = [] - backtesting._store_backtest_result("backtest-result.json", results, "DefStrat") + backtesting._store_backtest_result(Path("backtest-result.json"), results, "DefStrat") assert len(results) == 4 # Assert file_dump_json was only called once - assert names == ['backtest-result-DefStrat.json'] + assert names == [Path('backtest-result-DefStrat.json')] records = records[0] # Ensure records are of correct type assert len(records) == 4 diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 0317bb37d..1c4e2445c 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -618,3 +618,77 @@ def test_continue_hyperopt(mocker, default_conf, caplog): assert unlinkmock.call_count == 0 assert log_has(f"Continuing on previous hyperopt results.", caplog) + + +def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + parallel = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + ) + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'all', + 'hyperopt_jobs': 1, + 'print_json': True, + }) + + hyperopt = Hyperopt(default_conf) + hyperopt.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + hyperopt.start() + + parallel.assert_called_once() + + out, err = capsys.readouterr() + assert '{"params":{"mfi-value":null,"fastd-value":null,"adx-value":null,"rsi-value":null,"mfi-enabled":null,"fastd-enabled":null,"adx-enabled":null,"rsi-enabled":null,"trigger":null,"sell-mfi-value":null,"sell-fastd-value":null,"sell-adx-value":null,"sell-rsi-value":null,"sell-mfi-enabled":null,"sell-fastd-enabled":null,"sell-adx-enabled":null,"sell-rsi-enabled":null,"sell-trigger":null},"minimal_roi":{},"stoploss":null}' in out # noqa: E501 + assert dumper.called + # Should be called twice, once for tickerdata, once to save evaluations + assert dumper.call_count == 2 + + +def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> None: + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch( + 'freqtrade.optimize.hyperopt.get_timeframe', + MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) + ) + + parallel = mocker.patch( + 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + ) + patch_exchange(mocker) + + default_conf.update({'config': 'config.json.example', + 'epochs': 1, + 'timerange': None, + 'spaces': 'roi stoploss', + 'hyperopt_jobs': 1, + 'print_json': True, + }) + + hyperopt = Hyperopt(default_conf) + hyperopt.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) + + hyperopt.start() + + parallel.assert_called_once() + + out, err = capsys.readouterr() + assert '{"minimal_roi":{},"stoploss":null}' in out + assert dumper.called + # Should be called twice, once for tickerdata, once to save evaluations + assert dumper.call_count == 2 diff --git a/freqtrade/tests/rpc/test_rpc.py b/freqtrade/tests/rpc/test_rpc.py index d273244b0..5d3fb7920 100644 --- a/freqtrade/tests/rpc/test_rpc.py +++ b/freqtrade/tests/rpc/test_rpc.py @@ -44,7 +44,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: with pytest.raises(RPCException, match=r'.*no active trade*'): rpc._rpc_trade_status() - freqtradebot.create_trade() + freqtradebot.create_trades() results = rpc._rpc_trade_status() assert { 'trade_id': 1, @@ -116,7 +116,7 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: with pytest.raises(RPCException, match=r'.*no active order*'): rpc._rpc_status_table() - freqtradebot.create_trade() + freqtradebot.create_trades() result = rpc._rpc_status_table() assert 'instantly' in result['Since'].all() assert 'ETH/BTC' in result['Pair'].all() @@ -151,7 +151,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, rpc = RPC(freqtradebot) rpc._fiat_converter = CryptoToFiatConverter() # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() trade = Trade.query.first() assert trade @@ -208,7 +208,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) @@ -222,7 +222,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, trade.close_date = datetime.utcnow() trade.is_open = False - freqtradebot.create_trade() + freqtradebot.create_trades() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) @@ -292,7 +292,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, rpc = RPC(freqtradebot) # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) @@ -536,7 +536,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: msg = rpc._rpc_forcesell('all') assert msg == {'result': 'Created sell orders for all open trades.'} - freqtradebot.create_trade() + freqtradebot.create_trades() msg = rpc._rpc_forcesell('all') assert msg == {'result': 'Created sell orders for all open trades.'} @@ -570,7 +570,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: assert cancel_order_mock.call_count == 1 assert trade.amount == filled_amount - freqtradebot.create_trade() + freqtradebot.create_trades() trade = Trade.query.filter(Trade.id == '2').first() amount = trade.amount # make an limit-buy open trade, if there is no 'filled', don't sell it @@ -589,7 +589,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: assert cancel_order_mock.call_count == 2 assert trade.amount == amount - freqtradebot.create_trade() + freqtradebot.create_trades() # make an limit-sell open trade mocker.patch( 'freqtrade.exchange.Exchange.get_order', @@ -622,7 +622,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, rpc = RPC(freqtradebot) # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() trade = Trade.query.first() assert trade @@ -660,7 +660,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: assert counts["current"] == 0 # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() counts = rpc._rpc_count() assert counts["current"] == 1 diff --git a/freqtrade/tests/rpc/test_rpc_apiserver.py b/freqtrade/tests/rpc/test_rpc_apiserver.py index a218d5622..794343a98 100644 --- a/freqtrade/tests/rpc/test_rpc_apiserver.py +++ b/freqtrade/tests/rpc/test_rpc_apiserver.py @@ -275,7 +275,7 @@ def test_api_count(botclient, mocker, ticker, fee, markets): assert rc.json["max"] == 1.0 # Create some test data - ftbot.create_trade() + ftbot.create_trades() rc = client_get(client, f"{BASE_URI}/count") assert_response(rc) assert rc.json["current"] == 1.0 @@ -329,7 +329,7 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li assert len(rc.json) == 1 assert rc.json == {"error": "Error querying _profit: no closed trade"} - ftbot.create_trade() + ftbot.create_trades() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade @@ -418,7 +418,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets): assert_response(rc, 502) assert rc.json == {'error': 'Error querying _status: no active trade'} - ftbot.create_trade() + ftbot.create_trades() rc = client_get(client, f"{BASE_URI}/status") assert_response(rc) assert len(rc.json) == 1 @@ -548,7 +548,7 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets): assert_response(rc, 502) assert rc.json == {"error": "Error querying _forcesell: invalid argument"} - ftbot.create_trade() + ftbot.create_trades() rc = client_post(client, f"{BASE_URI}/forcesell", data='{"tradeid": "1"}') diff --git a/freqtrade/tests/rpc/test_rpc_telegram.py b/freqtrade/tests/rpc/test_rpc_telegram.py index 3575520ad..2f469643f 100644 --- a/freqtrade/tests/rpc/test_rpc_telegram.py +++ b/freqtrade/tests/rpc/test_rpc_telegram.py @@ -192,7 +192,7 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: # Create some test data for _ in range(3): - freqtradebot.create_trade() + freqtradebot.create_trades() telegram._status(bot=MagicMock(), update=update) assert msg_mock.call_count == 1 @@ -240,7 +240,7 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No msg_mock.reset_mock() # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() # Trigger status while we have a fulfilled order for the open trade telegram._status(bot=MagicMock(), update=update) @@ -292,7 +292,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) msg_mock.reset_mock() # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() telegram._status_table(bot=MagicMock(), update=update) @@ -308,6 +308,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, limit_sell_order, markets, mocker) -> None: patch_exchange(mocker) + default_conf['max_open_trades'] = 1 mocker.patch( 'freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0 @@ -331,7 +332,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() trade = Trade.query.first() assert trade @@ -357,9 +358,9 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, # Reset msg_mock msg_mock.reset_mock() + freqtradebot.config['max_open_trades'] = 2 # Add two other trades - freqtradebot.create_trade() - freqtradebot.create_trade() + freqtradebot.create_trades() trades = Trade.query.all() for trade in trades: @@ -438,7 +439,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, msg_mock.reset_mock() # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade @@ -733,7 +734,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() trade = Trade.query.first() assert trade @@ -784,7 +785,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() # Decrease the price and sell it mocker.patch.multiple( @@ -832,14 +833,13 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker markets=PropertyMock(return_value=markets), validate_pairs=MagicMock(return_value={}) ) - + default_conf['max_open_trades'] = 4 freqtradebot = FreqtradeBot(default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) # Create some test data - for _ in range(4): - freqtradebot.create_trade() + freqtradebot.create_trades() rpc_mock.reset_mock() update.message.text = '/forcesell all' @@ -983,7 +983,7 @@ def test_performance_handle(default_conf, update, ticker, fee, telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() trade = Trade.query.first() assert trade @@ -1028,7 +1028,7 @@ def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> Non freqtradebot.state = State.RUNNING # Create some test data - freqtradebot.create_trade() + freqtradebot.create_trades() msg_mock.reset_mock() telegram._count(bot=MagicMock(), update=update) diff --git a/freqtrade/tests/strategy/test_interface.py b/freqtrade/tests/strategy/test_interface.py index 0eb7630a1..36c9ffcd4 100644 --- a/freqtrade/tests/strategy/test_interface.py +++ b/freqtrade/tests/strategy/test_interface.py @@ -286,3 +286,19 @@ def test__analyze_ticker_internal_skip_analyze(ticker_history, mocker, caplog) - assert ret['sell'].sum() == 0 assert not log_has('TA Analysis Launched', caplog) assert log_has('Skipping TA Analysis for already analyzed candle', caplog) + + +def test_is_pair_locked(default_conf): + strategy = DefaultStrategy(default_conf) + # dict should be empty + assert not strategy._pair_locked_until + + pair = 'ETH/BTC' + assert not strategy.is_pair_locked(pair) + strategy.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + # ETH/BTC locked for 4 minutes + assert strategy.is_pair_locked(pair) + + # XRP/BTC should not be locked now + pair = 'XRP/BTC' + assert not strategy.is_pair_locked(pair) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index bf744f72b..601f41e63 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -3,8 +3,8 @@ import argparse import pytest -from freqtrade.configuration import Arguments, TimeRange -from freqtrade.configuration.arguments import ARGS_DOWNLOADER, ARGS_PLOT_DATAFRAME +from freqtrade.configuration import Arguments +from freqtrade.configuration.arguments import ARGS_PLOT_DATAFRAME from freqtrade.configuration.cli_options import check_int_positive @@ -50,10 +50,10 @@ def test_parse_args_verbose() -> None: def test_common_scripts_options() -> None: - arguments = Arguments(['-p', 'ETH/BTC'], '') - arguments._build_args(ARGS_DOWNLOADER) - args = arguments._parse_args() - assert args.pairs == 'ETH/BTC' + args = Arguments(['download-data', '-p', 'ETH/BTC', 'XRP/BTC'], '').get_parsed_arg() + + assert args.pairs == ['ETH/BTC', 'XRP/BTC'] + assert hasattr(args, "func") def test_parse_args_version() -> None: @@ -86,30 +86,6 @@ def test_parse_args_strategy_path_invalid() -> None: Arguments(['--strategy-path'], '').get_parsed_arg() -def test_parse_timerange_incorrect() -> None: - assert TimeRange(None, 'line', 0, -200) == Arguments.parse_timerange('-200') - assert TimeRange('line', None, 200, 0) == Arguments.parse_timerange('200-') - assert TimeRange('index', 'index', 200, 500) == Arguments.parse_timerange('200-500') - - assert TimeRange('date', None, 1274486400, 0) == Arguments.parse_timerange('20100522-') - assert TimeRange(None, 'date', 0, 1274486400) == Arguments.parse_timerange('-20100522') - timerange = Arguments.parse_timerange('20100522-20150730') - assert timerange == TimeRange('date', 'date', 1274486400, 1438214400) - - # Added test for unix timestamp - BTC genesis date - assert TimeRange('date', None, 1231006505, 0) == Arguments.parse_timerange('1231006505-') - assert TimeRange(None, 'date', 0, 1233360000) == Arguments.parse_timerange('-1233360000') - timerange = Arguments.parse_timerange('1231006505-1233360000') - assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange - - # TODO: Find solution for the following case (passing timestamp in ms) - timerange = Arguments.parse_timerange('1231006505000-1233360000000') - assert TimeRange('date', 'date', 1231006505, 1233360000) != timerange - - with pytest.raises(Exception, match=r'Incorrect syntax.*'): - Arguments.parse_timerange('-') - - def test_parse_args_backtesting_invalid() -> None: with pytest.raises(SystemExit, match=r'2'): Arguments(['backtesting --ticker-interval'], '').get_parsed_arg() @@ -159,14 +135,14 @@ def test_parse_args_hyperopt_custom() -> None: def test_download_data_options() -> None: args = [ - '--pairs-file', 'file_with_pairs', '--datadir', 'datadir/directory', + 'download-data', + '--pairs-file', 'file_with_pairs', '--days', '30', '--exchange', 'binance' ] - arguments = Arguments(args, '') - arguments._build_args(ARGS_DOWNLOADER) - args = arguments._parse_args() + args = Arguments(args, '').get_parsed_arg() + assert args.pairs_file == 'file_with_pairs' assert args.datadir == 'datadir/directory' assert args.days == 30 @@ -186,7 +162,7 @@ def test_plot_dataframe_options() -> None: assert pargs.indicators1 == "sma10,sma100" assert pargs.indicators2 == "macd,fastd,fastk" assert pargs.plot_limit == 30 - assert pargs.pairs == "UNITTEST/BTC" + assert pargs.pairs == ["UNITTEST/BTC"] def test_check_int_positive() -> None: diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 2bc561b7d..5cfee0698 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -2,7 +2,6 @@ import json import logging import warnings -from argparse import Namespace from copy import deepcopy from pathlib import Path from unittest.mock import MagicMock @@ -11,10 +10,10 @@ import pytest from jsonschema import Draft4Validator, ValidationError, validate from freqtrade import OperationalException, constants -from freqtrade.configuration import Arguments, Configuration +from freqtrade.configuration import Arguments, Configuration, validate_config_consistency from freqtrade.configuration.check_exchange import check_exchange +from freqtrade.configuration.config_validation import validate_config_schema from freqtrade.configuration.create_datadir import create_datadir -from freqtrade.configuration.json_schema import validate_config_schema from freqtrade.configuration.load_config import load_config_file from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.loggers import _set_loggers @@ -625,21 +624,45 @@ def test_validate_tsl(default_conf): with pytest.raises(OperationalException, match=r'The config trailing_only_offset_is_reached needs ' 'trailing_stop_positive_offset to be more than 0 in your config.'): - configuration = Configuration(Namespace()) - configuration._validate_config_consistency(default_conf) + validate_config_consistency(default_conf) default_conf['trailing_stop_positive_offset'] = 0.01 default_conf['trailing_stop_positive'] = 0.015 with pytest.raises(OperationalException, match=r'The config trailing_stop_positive_offset needs ' 'to be greater than trailing_stop_positive_offset in your config.'): - configuration = Configuration(Namespace()) - configuration._validate_config_consistency(default_conf) + validate_config_consistency(default_conf) default_conf['trailing_stop_positive'] = 0.01 default_conf['trailing_stop_positive_offset'] = 0.015 - Configuration(Namespace()) - configuration._validate_config_consistency(default_conf) + validate_config_consistency(default_conf) + + +def test_validate_edge(edge_conf): + edge_conf.update({"pairlist": { + "method": "VolumePairList", + }}) + + with pytest.raises(OperationalException, + match="Edge and VolumePairList are incompatible, " + "Edge will override whatever pairs VolumePairlist selects."): + validate_config_consistency(edge_conf) + + edge_conf.update({"pairlist": { + "method": "StaticPairList", + }}) + validate_config_consistency(edge_conf) + + +def test_load_config_test_comments() -> None: + """ + Load config with comments + """ + config_file = Path(__file__).parents[0] / "config_test_comments.json" + print(config_file) + conf = load_config_file(str(config_file)) + + assert conf def test_load_config_default_exchange(all_conf) -> None: @@ -693,3 +716,109 @@ def test_load_config_default_subkeys(all_conf, keys) -> None: validate_config_schema(all_conf) assert subkey in all_conf[key] assert all_conf[key][subkey] == keys[2] + + +def test_pairlist_resolving(): + arglist = [ + 'download-data', + '--pairs', 'ETH/BTC', 'XRP/BTC', + '--exchange', 'binance' + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] + assert config['exchange']['name'] == 'binance' + + +def test_pairlist_resolving_with_config(mocker, default_conf): + patched_configuration_load_config_file(mocker, default_conf) + arglist = [ + '--config', 'config.json', + 'download-data', + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert config['pairs'] == default_conf['exchange']['pair_whitelist'] + assert config['exchange']['name'] == default_conf['exchange']['name'] + + # Override pairs + arglist = [ + '--config', 'config.json', + 'download-data', + '--pairs', 'ETH/BTC', 'XRP/BTC', + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] + assert config['exchange']['name'] == default_conf['exchange']['name'] + + +def test_pairlist_resolving_with_config_pl(mocker, default_conf): + patched_configuration_load_config_file(mocker, default_conf) + load_mock = mocker.patch("freqtrade.configuration.configuration.json_load", + MagicMock(return_value=['XRP/BTC', 'ETH/BTC'])) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + + arglist = [ + '--config', 'config.json', + 'download-data', + '--pairs-file', 'pairs.json', + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert load_mock.call_count == 1 + assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] + assert config['exchange']['name'] == default_conf['exchange']['name'] + + +def test_pairlist_resolving_with_config_pl_not_exists(mocker, default_conf): + patched_configuration_load_config_file(mocker, default_conf) + mocker.patch("freqtrade.configuration.configuration.json_load", + MagicMock(return_value=['XRP/BTC', 'ETH/BTC'])) + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + + arglist = [ + '--config', 'config.json', + 'download-data', + '--pairs-file', 'pairs.json', + ] + + args = Arguments(arglist, '').get_parsed_arg() + + with pytest.raises(OperationalException, match=r"No pairs file found with path.*"): + configuration = Configuration(args) + configuration.get_config() + + +def test_pairlist_resolving_fallback(mocker): + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch("freqtrade.configuration.configuration.json_load", + MagicMock(return_value=['XRP/BTC', 'ETH/BTC'])) + arglist = [ + 'download-data', + '--exchange', 'binance' + ] + + args = Arguments(arglist, '').get_parsed_arg() + + configuration = Configuration(args) + config = configuration.get_config() + + assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] + assert config['exchange']['name'] == 'binance' diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 4e649250a..24d070d2d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -253,13 +253,13 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, assert result == default_conf['stake_amount'] / conf['max_open_trades'] # create one trade, order amount should be 'balance / (max_open_trades - num_open_trades)' - freqtrade.create_trade() + freqtrade.execute_buy('ETH/BTC', result) result = freqtrade._get_trade_stake_amount('LTC/BTC') assert result == default_conf['stake_amount'] / (conf['max_open_trades'] - 1) # create 2 trades, order amount should be None - freqtrade.create_trade() + freqtrade.execute_buy('LTC/BTC', result) result = freqtrade._get_trade_stake_amount('XRP/BTC') assert result is None @@ -301,6 +301,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) + edge_conf['max_open_trades'] = float('inf') # Strategy stoploss is -0.1 but Edge imposes a stoploss at -0.2 # Thus, if price falls 21%, stoploss should be triggered @@ -325,7 +326,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.update(limit_buy_order) ############################################# @@ -341,6 +342,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) + edge_conf['max_open_trades'] = float('inf') # Strategy stoploss is -0.1 but Edge imposes a stoploss at -0.2 # Thus, if price falls 15%, stoploss should not be triggered @@ -365,7 +367,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.update(limit_buy_order) ############################################# @@ -379,6 +381,7 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, patch_RPCManager(mocker) patch_exchange(mocker) default_conf['stake_amount'] = 0.0000098751 + default_conf['max_open_trades'] = 2 mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, @@ -388,7 +391,7 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert trade is not None @@ -396,7 +399,7 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, assert trade.is_open assert trade.open_date is not None - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.order_by(Trade.id.desc()).first() assert trade is not None @@ -519,7 +522,7 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: assert result == min(8, 2 * 2) / 0.9 -def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: +def test_create_trades(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -534,7 +537,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocke whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert trade is not None @@ -552,8 +555,8 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, markets, mocke assert whitelist == default_conf['exchange']['pair_whitelist'] -def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, - fee, markets, mocker) -> None: +def test_create_trades_no_stake_amount(default_conf, ticker, limit_buy_order, + fee, markets, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_wallet(mocker, free=default_conf['stake_amount'] * 0.5) @@ -568,11 +571,11 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, patch_get_signal(freqtrade) with pytest.raises(DependencyException, match=r'.*stake amount.*'): - freqtrade.create_trade() + freqtrade.create_trades() -def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, - fee, markets, mocker) -> None: +def test_create_trades_minimal_amount(default_conf, ticker, limit_buy_order, + fee, markets, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) @@ -587,13 +590,13 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trade() + freqtrade.create_trades() rate, amount = buy_mock.call_args[1]['rate'], buy_mock.call_args[1]['amount'] assert rate * amount >= default_conf['stake_amount'] -def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order, - fee, markets, mocker) -> None: +def test_create_trades_too_small_stake_amount(default_conf, ticker, limit_buy_order, + fee, markets, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) @@ -609,11 +612,11 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - assert not freqtrade.create_trade() + assert not freqtrade.create_trades() -def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, - fee, markets, mocker) -> None: +def test_create_trades_limit_reached(default_conf, ticker, limit_buy_order, + fee, markets, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -630,12 +633,12 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - assert not freqtrade.create_trade() + assert not freqtrade.create_trades() assert freqtrade._get_trade_stake_amount('ETH/BTC') is None -def test_create_trade_no_pairs_let(default_conf, ticker, limit_buy_order, fee, - markets, mocker, caplog) -> None: +def test_create_trades_no_pairs_let(default_conf, ticker, limit_buy_order, fee, + markets, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -650,13 +653,13 @@ def test_create_trade_no_pairs_let(default_conf, ticker, limit_buy_order, fee, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - assert freqtrade.create_trade() - assert not freqtrade.create_trade() + assert freqtrade.create_trades() + assert not freqtrade.create_trades() assert log_has("No currency pair in whitelist, but checking to sell open trades.", caplog) -def test_create_trade_no_pairs_in_whitelist(default_conf, ticker, limit_buy_order, fee, - markets, mocker, caplog) -> None: +def test_create_trades_no_pairs_in_whitelist(default_conf, ticker, limit_buy_order, fee, + markets, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -670,11 +673,11 @@ def test_create_trade_no_pairs_in_whitelist(default_conf, ticker, limit_buy_orde freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - assert not freqtrade.create_trade() + assert not freqtrade.create_trades() assert log_has("Whitelist is empty.", caplog) -def test_create_trade_no_signal(default_conf, fee, mocker) -> None: +def test_create_trades_no_signal(default_conf, fee, mocker) -> None: default_conf['dry_run'] = True patch_RPCManager(mocker) @@ -690,7 +693,56 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: Trade.query = MagicMock() Trade.query.filter = MagicMock() - assert not freqtrade.create_trade() + assert not freqtrade.create_trades() + + +@pytest.mark.parametrize("max_open", range(0, 5)) +def test_create_trades_multiple_trades(default_conf, ticker, + fee, markets, mocker, max_open) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + default_conf['max_open_trades'] = max_open + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker, + buy=MagicMock(return_value={'id': "12355555"}), + get_fee=fee, + markets=PropertyMock(return_value=markets) + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + + freqtrade.create_trades() + + trades = Trade.get_open_trades() + assert len(trades) == max_open + + +def test_create_trades_preopen(default_conf, ticker, fee, markets, mocker) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + default_conf['max_open_trades'] = 4 + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker, + buy=MagicMock(return_value={'id': "12355555"}), + get_fee=fee, + markets=PropertyMock(return_value=markets) + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + + # Create 2 existing trades + freqtrade.execute_buy('ETH/BTC', default_conf['stake_amount']) + freqtrade.execute_buy('NEO/BTC', default_conf['stake_amount']) + + assert len(Trade.get_open_trades()) == 2 + + # Create 2 new trades using create_trades + assert freqtrade.create_trades() + + trades = Trade.get_open_trades() + assert len(trades) == 4 def test_process_trade_creation(default_conf, ticker, limit_buy_order, @@ -711,8 +763,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert not trades - result = freqtrade.process() - assert result is True + freqtrade.process() trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert len(trades) == 1 @@ -744,8 +795,7 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non worker = Worker(args=None, config=default_conf) patch_get_signal(worker.freqtrade) - result = worker._process() - assert result is False + worker._process() assert sleep_mock.has_calls() @@ -763,8 +813,7 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) -> assert worker.state == State.RUNNING - result = worker._process() - assert result is False + worker._process() assert worker.state == State.STOPPED assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] @@ -786,13 +835,14 @@ def test_process_trade_handling( trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert not trades - result = freqtrade.process() - assert result is True + freqtrade.process() + trades = Trade.query.filter(Trade.is_open.is_(True)).all() assert len(trades) == 1 - result = freqtrade.process() - assert result is False + # Nothing happened ... + freqtrade.process() + assert len(trades) == 1 def test_process_trade_no_whitelist_pair( @@ -834,11 +884,10 @@ def test_process_trade_no_whitelist_pair( )) assert pair not in freqtrade.active_pair_whitelist - result = freqtrade.process() + freqtrade.process() assert pair in freqtrade.active_pair_whitelist # Make sure each pair is only in the list once assert len(freqtrade.active_pair_whitelist) == len(set(freqtrade.active_pair_whitelist)) - assert result is True def test_process_informative_pairs_added(default_conf, ticker, markets, mocker) -> None: @@ -1078,7 +1127,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, # Fourth case: when stoploss is set and it is hit # should unset stoploss_order_id and return true # as a trade actually happened - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.is_open = True trade.open_order_id = None @@ -1153,7 +1202,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, patch_get_signal(freqtrade) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.is_open = True trade.open_order_id = None @@ -1243,7 +1292,7 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c # setting stoploss_on_exchange_interval to 60 seconds freqtrade.strategy.order_types['stoploss_on_exchange_interval'] = 60 patch_get_signal(freqtrade) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.is_open = True trade.open_order_id = None @@ -1286,7 +1335,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) - + edge_conf['max_open_trades'] = float('inf') mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=MagicMock(return_value={ @@ -1324,7 +1373,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, freqtrade.active_pair_whitelist = freqtrade.edge.adjust(freqtrade.active_pair_whitelist) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.is_open = True trade.open_order_id = None @@ -1388,21 +1437,19 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, stop_price=0.00002344 * 0.99) -def test_process_maybe_execute_buy(mocker, default_conf) -> None: +def test_process_maybe_execute_buy(mocker, default_conf, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade', MagicMock(return_value=True)) - assert freqtrade.process_maybe_execute_buy() - - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade', MagicMock(return_value=False)) - assert not freqtrade.process_maybe_execute_buy() + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trades', MagicMock(return_value=False)) + freqtrade.process_maybe_execute_buy() + assert log_has('Found no buy signals for whitelisted currencies. Trying again...', caplog) def test_process_maybe_execute_buy_exception(mocker, default_conf, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch( - 'freqtrade.freqtradebot.FreqtradeBot.create_trade', + 'freqtrade.freqtradebot.FreqtradeBot.create_trades', MagicMock(side_effect=DependencyException) ) freqtrade.process_maybe_execute_buy() @@ -1589,7 +1636,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert trade @@ -1629,7 +1676,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, patch_get_signal(freqtrade, value=(True, True)) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trade() + freqtrade.create_trades() # Buy and Sell triggering, so doing nothing ... trades = Trade.query.all() @@ -1638,7 +1685,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, # Buy is triggering, so buying ... patch_get_signal(freqtrade, value=(True, False)) - freqtrade.create_trade() + freqtrade.create_trades() trades = Trade.query.all() nb_trades = len(trades) assert nb_trades == 1 @@ -1685,7 +1732,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, patch_get_signal(freqtrade, value=(True, False)) freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.is_open = True @@ -1717,7 +1764,7 @@ def test_handle_trade_experimental( freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.is_open = True @@ -1745,7 +1792,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, patch_get_signal(freqtrade) # Create trade and sell it - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert trade @@ -2085,7 +2132,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert trade @@ -2131,7 +2178,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert trade @@ -2180,7 +2227,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert trade @@ -2237,7 +2284,7 @@ def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee, freqtrade.strategy.order_types['stoploss_on_exchange'] = True patch_get_signal(freqtrade) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() Trade.session = MagicMock() @@ -2284,7 +2331,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert trade @@ -2336,7 +2383,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() freqtrade.process_maybe_execute_sell(trade) assert trade @@ -2374,8 +2421,8 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, assert rpc_mock.call_count == 2 -def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, - ticker_sell_up, markets, mocker) -> None: +def test_execute_sell_market_order(default_conf, ticker, fee, + ticker_sell_up, markets, mocker) -> None: rpc_mock = patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -2388,7 +2435,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert trade @@ -2398,10 +2445,13 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, 'freqtrade.exchange.Exchange', get_ticker=ticker_sell_up ) - freqtrade.config = {} + freqtrade.config['order_types']['sell'] = 'market' freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI) + assert not trade.is_open + assert trade.close_profit == 0.0611052 + assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] assert { @@ -2411,63 +2461,18 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee, 'gain': 'profit', 'limit': 1.172e-05, 'amount': 90.99181073703367, - 'order_type': 'limit', + 'order_type': 'market', 'open_rate': 1.099e-05, 'current_rate': 1.172e-05, 'profit_amount': 6.126e-05, 'profit_percent': 0.0611052, + 'stake_currency': 'BTC', + 'fiat_currency': 'USD', 'sell_reason': SellType.ROI.value } == last_msg -def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee, - ticker_sell_down, markets, mocker) -> None: - rpc_mock = patch_RPCManager(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - _load_markets=MagicMock(return_value={}), - get_ticker=ticker, - get_fee=fee, - markets=PropertyMock(return_value=markets) - ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) - - # Create some test data - freqtrade.create_trade() - - trade = Trade.query.first() - assert trade - - # Decrease the price and sell it - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_down - ) - - freqtrade.config = {} - freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], - sell_reason=SellType.STOP_LOSS) - - assert rpc_mock.call_count == 2 - last_msg = rpc_mock.call_args_list[-1][0][0] - assert { - 'type': RPCMessageType.SELL_NOTIFICATION, - 'exchange': 'Bittrex', - 'pair': 'ETH/BTC', - 'gain': 'loss', - 'limit': 1.044e-05, - 'amount': 90.99181073703367, - 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.044e-05, - 'profit_amount': -5.492e-05, - 'profit_percent': -0.05478342, - 'sell_reason': SellType.STOP_LOSS.value - } == last_msg - - def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) @@ -2491,7 +2496,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2522,7 +2527,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2553,7 +2558,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market patch_get_signal(freqtrade) freqtrade.strategy.stop_loss_reached = MagicMock(return_value=SellCheckTuple( sell_flag=False, sell_type=SellType.NONE)) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2584,7 +2589,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2593,6 +2598,43 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke assert trade.sell_reason == SellType.SELL_SIGNAL.value +def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, markets, mocker, caplog) -> None: + patch_RPCManager(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + _load_markets=MagicMock(return_value={}), + get_ticker=ticker, + get_fee=fee, + markets=PropertyMock(return_value=markets) + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + + # Create some test data + freqtrade.create_trades() + + trade = Trade.query.first() + assert trade + + # Decrease the price and sell it + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=ticker_sell_down + ) + + freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], + sell_reason=SellType.STOP_LOSS) + trade.close(ticker_sell_down()['bid']) + assert trade.pair in freqtrade.strategy._pair_locked_until + assert freqtrade.strategy.is_pair_locked(trade.pair) + + # reinit - should buy other pair. + caplog.clear() + freqtrade.create_trades() + + assert log_has(f"Pair {trade.pair} is currently locked.", caplog) + + def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -2614,7 +2656,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2646,7 +2688,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert freqtrade.handle_trade(trade) is False @@ -2699,7 +2741,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2757,7 +2799,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2820,7 +2862,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2879,7 +2921,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() trade.update(limit_buy_order) @@ -3136,7 +3178,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert trade is not None @@ -3170,7 +3212,7 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o # Save state of current whitelist freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert trade is None @@ -3276,7 +3318,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trade() + freqtrade.create_trades() trade = Trade.query.first() assert trade diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index d8ec532b0..409025a3c 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -1,7 +1,7 @@ # pragma pylint: disable=missing-docstring from copy import deepcopy -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock import pytest @@ -21,6 +21,7 @@ def test_parse_args_backtesting(mocker) -> None: further argument parsing is done in test_arguments.py """ backtesting_mock = mocker.patch('freqtrade.optimize.start_backtesting', MagicMock()) + backtesting_mock.__name__ = PropertyMock("start_backtesting") # it's sys.exit(0) at the end of backtesting with pytest.raises(SystemExit): main(['backtesting']) @@ -36,6 +37,7 @@ def test_parse_args_backtesting(mocker) -> None: def test_main_start_hyperopt(mocker) -> None: hyperopt_mock = mocker.patch('freqtrade.optimize.start_hyperopt', MagicMock()) + hyperopt_mock.__name__ = PropertyMock("start_hyperopt") # it's sys.exit(0) at the end of hyperopt with pytest.raises(SystemExit): main(['hyperopt']) diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 1a6b2a92d..c55083e64 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring,C0103 import datetime +from pathlib import Path from unittest.mock import MagicMock from freqtrade.data.converter import parse_ticker_dataframe @@ -34,12 +35,12 @@ def test_datesarray_to_datetimearray(ticker_history_list): def test_file_dump_json(mocker) -> None: file_open = mocker.patch('freqtrade.misc.open', MagicMock()) json_dump = mocker.patch('rapidjson.dump', MagicMock()) - file_dump_json('somefile', [1, 2, 3]) + file_dump_json(Path('somefile'), [1, 2, 3]) assert file_open.call_count == 1 assert json_dump.call_count == 1 file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock()) json_dump = mocker.patch('rapidjson.dump', MagicMock()) - file_dump_json('somefile', [1, 2, 3], True) + file_dump_json(Path('somefile'), [1, 2, 3], True) assert file_open.call_count == 1 assert json_dump.call_count == 1 diff --git a/freqtrade/tests/test_plotting.py b/freqtrade/tests/test_plotting.py index bfdd72215..94d40ab84 100644 --- a/freqtrade/tests/test_plotting.py +++ b/freqtrade/tests/test_plotting.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock import plotly.graph_objects as go from plotly.subplots import make_subplots -from freqtrade.configuration import Arguments, TimeRange +from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data from freqtrade.plot.plotting import (add_indicators, add_profit, @@ -50,7 +50,7 @@ def test_init_plotscript(default_conf, mocker): assert "pairs" in ret assert "strategy" in ret - default_conf['pairs'] = "POWR/BTC,XLM/BTC" + default_conf['pairs'] = ["POWR/BTC", "XLM/BTC"] ret = init_plotscript(default_conf) assert "tickers" in ret assert "POWR/BTC" in ret["tickers"] @@ -222,7 +222,7 @@ def test_generate_plot_file(mocker, caplog): def test_add_profit(): filename = history.make_testdata_path(None) / "backtest-result_test.json" bt_data = load_backtest_data(filename) - timerange = Arguments.parse_timerange("20180110-20180112") + timerange = TimeRange.parse_timerange("20180110-20180112") df = history.load_pair_history(pair="POWR/BTC", ticker_interval='5m', datadir=None, timerange=timerange) @@ -242,7 +242,7 @@ def test_add_profit(): def test_generate_profit_graph(): filename = history.make_testdata_path(None) / "backtest-result_test.json" trades = load_backtest_data(filename) - timerange = Arguments.parse_timerange("20180110-20180112") + timerange = TimeRange.parse_timerange("20180110-20180112") pairs = ["POWR/BTC", "XLM/BTC"] tickers = history.load_data(datadir=None, diff --git a/freqtrade/tests/test_timerange.py b/freqtrade/tests/test_timerange.py new file mode 100644 index 000000000..6599472fb --- /dev/null +++ b/freqtrade/tests/test_timerange.py @@ -0,0 +1,28 @@ +# pragma pylint: disable=missing-docstring, C0103 +import pytest + +from freqtrade.configuration import TimeRange + + +def test_parse_timerange_incorrect() -> None: + assert TimeRange(None, 'line', 0, -200) == TimeRange.parse_timerange('-200') + assert TimeRange('line', None, 200, 0) == TimeRange.parse_timerange('200-') + assert TimeRange('index', 'index', 200, 500) == TimeRange.parse_timerange('200-500') + + assert TimeRange('date', None, 1274486400, 0) == TimeRange.parse_timerange('20100522-') + assert TimeRange(None, 'date', 0, 1274486400) == TimeRange.parse_timerange('-20100522') + timerange = TimeRange.parse_timerange('20100522-20150730') + assert timerange == TimeRange('date', 'date', 1274486400, 1438214400) + + # Added test for unix timestamp - BTC genesis date + assert TimeRange('date', None, 1231006505, 0) == TimeRange.parse_timerange('1231006505-') + assert TimeRange(None, 'date', 0, 1233360000) == TimeRange.parse_timerange('-1233360000') + timerange = TimeRange.parse_timerange('1231006505-1233360000') + assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange + + # TODO: Find solution for the following case (passing timestamp in ms) + timerange = TimeRange.parse_timerange('1231006505000-1233360000000') + assert TimeRange('date', 'date', 1231006505, 1233360000) != timerange + + with pytest.raises(Exception, match=r'Incorrect syntax.*'): + TimeRange.parse_timerange('-') diff --git a/freqtrade/tests/test_utils.py b/freqtrade/tests/test_utils.py index a12b709d7..003b3b286 100644 --- a/freqtrade/tests/test_utils.py +++ b/freqtrade/tests/test_utils.py @@ -1,8 +1,13 @@ -from freqtrade.utils import setup_utils_configuration, start_list_exchanges -from freqtrade.tests.conftest import get_args -from freqtrade.state import RunMode - import re +from pathlib import Path +from unittest.mock import MagicMock, PropertyMock + +import pytest + +from freqtrade.state import RunMode +from freqtrade.tests.conftest import get_args, log_has, patch_exchange +from freqtrade.utils import (setup_utils_configuration, start_download_data, + start_list_exchanges) def test_setup_utils_configuration(): @@ -40,3 +45,87 @@ def test_list_exchanges(capsys): assert not re.match(r"Exchanges supported by ccxt and available.*", captured.out) assert re.search(r"^binance$", captured.out, re.MULTILINE) assert re.search(r"^bittrex$", captured.out, re.MULTILINE) + + +def test_download_data(mocker, markets, caplog): + dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch.object(Path, "unlink", MagicMock()) + + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--erase", + ] + start_download_data(get_args(args)) + + assert dl_mock.call_count == 4 + assert dl_mock.call_args[1]['timerange'].starttype is None + assert dl_mock.call_args[1]['timerange'].stoptype is None + assert log_has("Deleting existing data for pair ETH/BTC, interval 1m.", caplog) + assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) + + +def test_download_data_days(mocker, markets, caplog): + dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch.object(Path, "unlink", MagicMock()) + + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20", + ] + + start_download_data(get_args(args)) + + assert dl_mock.call_count == 4 + assert dl_mock.call_args[1]['timerange'].starttype == 'date' + + assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) + + +def test_download_data_no_markets(mocker, caplog): + dl_mock = mocker.patch('freqtrade.utils.download_pair_history', MagicMock()) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) + ) + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + ] + start_download_data(get_args(args)) + + assert dl_mock.call_count == 0 + assert log_has("Skipping pair ETH/BTC...", caplog) + assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) + + +def test_download_data_keyboardInterrupt(mocker, caplog, markets): + dl_mock = mocker.patch('freqtrade.utils.download_pair_history', + MagicMock(side_effect=KeyboardInterrupt)) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + ] + with pytest.raises(SystemExit): + start_download_data(get_args(args)) + + assert dl_mock.call_count == 1 diff --git a/freqtrade/utils.py b/freqtrade/utils.py index d550ef43c..002d89738 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -1,11 +1,16 @@ import logging +import sys from argparse import Namespace +from pathlib import Path from typing import Any, Dict -from freqtrade.configuration import Configuration -from freqtrade.exchange import available_exchanges -from freqtrade.state import RunMode +import arrow +from freqtrade.configuration import Configuration, TimeRange +from freqtrade.data.history import download_pair_history +from freqtrade.exchange import available_exchanges +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -17,7 +22,7 @@ def setup_utils_configuration(args: Namespace, method: RunMode) -> Dict[str, Any :return: Configuration """ configuration = Configuration(args, method) - config = configuration.load_config() + config = configuration.get_config() config['exchange']['dry_run'] = True # Ensure we do not use Exchange credentials @@ -39,3 +44,56 @@ def start_list_exchanges(args: Namespace) -> None: else: print(f"Exchanges supported by ccxt and available for Freqtrade: " f"{', '.join(available_exchanges())}") + + +def start_download_data(args: Namespace) -> None: + """ + Download data (former download_backtest_data.py script) + """ + config = setup_utils_configuration(args, RunMode.OTHER) + + timerange = TimeRange() + if 'days' in config: + time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + + dl_path = Path(config['datadir']) + logger.info(f'About to download pairs: {config["pairs"]}, ' + f'intervals: {config["timeframes"]} to {dl_path}') + + pairs_not_available = [] + + try: + # Init exchange + exchange = ExchangeResolver(config['exchange']['name'], config).exchange + + for pair in config["pairs"]: + if pair not in exchange.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + for ticker_interval in config["timeframes"]: + pair_print = pair.replace('/', '_') + filename = f'{pair_print}-{ticker_interval}.json' + dl_file = dl_path.joinpath(filename) + if config.get("erase") and dl_file.exists(): + logger.info( + f'Deleting existing data for pair {pair}, interval {ticker_interval}.') + dl_file.unlink() + + logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') + download_pair_history(datadir=dl_path, exchange=exchange, + pair=pair, ticker_interval=str(ticker_interval), + timerange=timerange) + + except KeyboardInterrupt: + sys.exit("SIGINT received, aborting ...") + + finally: + if pairs_not_available: + logger.info( + f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {config['exchange']['name']}.") + + # configuration.resolve_pairs_list() + print(config) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index db0dba0e8..df792e35e 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -127,11 +127,10 @@ class Worker(object): time.sleep(duration) return result - def _process(self) -> bool: + def _process(self) -> None: logger.debug("========================================") - state_changed = False try: - state_changed = self.freqtrade.process() + self.freqtrade.process() except TemporaryError as error: logger.warning(f"Error: {error}, retrying in {constants.RETRY_TIMEOUT} seconds...") time.sleep(constants.RETRY_TIMEOUT) @@ -144,10 +143,6 @@ class Worker(object): }) logger.exception('OperationalException. Stopping trader ...') self.freqtrade.state = State.STOPPED - # TODO: The return value of _process() is not used apart tests - # and should (could) be eliminated later. See PR #1689. -# state_changed = True - return state_changed def _reconfigure(self) -> None: """ diff --git a/requirements-common.txt b/requirements-common.txt index 651be7611..3d80c3ef5 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,7 +1,7 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.1043 -SQLAlchemy==1.3.6 +ccxt==1.18.1068 +SQLAlchemy==1.3.7 python-telegram-bot==11.1.0 arrow==0.14.5 cachetools==3.1.1 diff --git a/requirements-dev.txt b/requirements-dev.txt index 03b37417e..6436c60e4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ flake8==3.7.8 flake8-type-annotations==0.1.0 flake8-tidy-imports==2.0.0 mypy==0.720 -pytest==5.0.1 +pytest==5.1.0 pytest-asyncio==0.10.0 pytest-cov==2.7.1 pytest-mock==1.10.4 diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index 580592294..496f83c7d 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -1,144 +1,9 @@ -#!/usr/bin/env python3 -""" -This script generates json files with pairs history data -""" -import arrow -import json import sys -from pathlib import Path -from typing import Any, Dict, List -from freqtrade.configuration import Arguments, TimeRange -from freqtrade.configuration import Configuration -from freqtrade.configuration.arguments import ARGS_DOWNLOADER -from freqtrade.configuration.check_exchange import check_exchange -from freqtrade.configuration.load_config import load_config_file -from freqtrade.data.history import download_pair_history -from freqtrade.exchange import Exchange -from freqtrade.misc import deep_merge_dicts -import logging +print("This script has been integrated into freqtrade " + "and it's functionality is available by calling `freqtrade download-data`.") +print("Please check the documentation on https://www.freqtrade.io/en/latest/backtesting/ " + "for details.") -logger = logging.getLogger('download_backtest_data') - -DEFAULT_DL_PATH = 'user_data/data' - -# Do not read the default config if config is not specified -# in the command line options explicitely -arguments = Arguments(sys.argv[1:], 'Download backtest data', - no_default_config=True) -arguments._build_args(optionlist=ARGS_DOWNLOADER) -args = arguments._parse_args() - -# Use bittrex as default exchange -exchange_name = args.exchange or 'bittrex' - -pairs: List = [] - -configuration = Configuration(args) -config: Dict[str, Any] = {} - -if args.config: - # Now expecting a list of config filenames here, not a string - for path in args.config: - logger.info(f"Using config: {path}...") - # Merge config options, overwriting old values - config = deep_merge_dicts(load_config_file(path), config) - - config['stake_currency'] = '' - # Ensure we do not use Exchange credentials - config['exchange']['dry_run'] = True - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - - pairs = config['exchange']['pair_whitelist'] - - if config.get('ticker_interval'): - timeframes = args.timeframes or [config.get('ticker_interval')] - else: - timeframes = args.timeframes or ['1m', '5m'] - -else: - config = { - 'stake_currency': '', - 'dry_run': True, - 'exchange': { - 'name': exchange_name, - 'key': '', - 'secret': '', - 'pair_whitelist': [], - 'ccxt_async_config': { - 'enableRateLimit': True, - 'rateLimit': 200 - } - } - } - timeframes = args.timeframes or ['1m', '5m'] - -configuration._process_logging_options(config) - -if args.config and args.exchange: - logger.warning("The --exchange option is ignored, " - "using exchange settings from the configuration file.") - -# Check if the exchange set by the user is supported -check_exchange(config) - -configuration._process_datadir_options(config) - -dl_path = Path(config['datadir']) - -pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json') - -if not pairs or args.pairs_file: - logger.info(f'Reading pairs file "{pairs_file}".') - # Download pairs from the pairs file if no config is specified - # or if pairs file is specified explicitely - if not pairs_file.exists(): - sys.exit(f'No pairs file found with path "{pairs_file}".') - - with pairs_file.open() as file: - pairs = list(set(json.load(file))) - - pairs.sort() - -timerange = TimeRange() -if args.days: - time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d") - timerange = arguments.parse_timerange(f'{time_since}-') - -logger.info(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}') - -pairs_not_available = [] - -try: - # Init exchange - exchange = Exchange(config) - - for pair in pairs: - if pair not in exchange._api.markets: - pairs_not_available.append(pair) - logger.info(f"Skipping pair {pair}...") - continue - for ticker_interval in timeframes: - pair_print = pair.replace('/', '_') - filename = f'{pair_print}-{ticker_interval}.json' - dl_file = dl_path.joinpath(filename) - if args.erase and dl_file.exists(): - logger.info( - f'Deleting existing data for pair {pair}, interval {ticker_interval}.') - dl_file.unlink() - - logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') - download_pair_history(datadir=dl_path, exchange=exchange, - pair=pair, ticker_interval=str(ticker_interval), - timerange=timerange) - -except KeyboardInterrupt: - sys.exit("SIGINT received, aborting ...") - -finally: - if pairs_not_available: - logger.info( - f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {config['exchange']['name']}.") +sys.exit(1)