diff --git a/config_full.json.example b/config_full.json.example index c6b229ea3..5789e49ac 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -119,7 +119,8 @@ "initial_state": "running", "forcebuy_enable": false, "internals": { - "process_throttle_secs": 5 + "process_throttle_secs": 5, + "heartbeat_interval": 60 }, "strategy": "DefaultStrategy", "strategy_path": "user_data/strategies/" diff --git a/docs/advanced-setup.md b/docs/advanced-setup.md new file mode 100644 index 000000000..e6334d2c1 --- /dev/null +++ b/docs/advanced-setup.md @@ -0,0 +1,33 @@ +# Advanced Post-installation Tasks + +This page explains some advanced tasks and configuration options that can be performed after the bot installation and may be uselful in some environments. + +If you do not know what things mentioned here mean, you probably do not need it. + +## Configure the bot running as a systemd service + +Copy the `freqtrade.service` file to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. + +After that you can start the daemon with: + +```bash +systemctl --user start freqtrade +``` + +For this to be persistent (run when user is logged out) you'll need to enable `linger` for your freqtrade user. + +```bash +sudo loginctl enable-linger "$USER" +``` + +If you run the bot as a service, you can use systemd service manager as a software watchdog monitoring freqtrade bot +state and restarting it in the case of failures. If the `internals.sd_notify` parameter is set to true in the +configuration or the `--sd-notify` command line option is used, the bot will send keep-alive ping messages to systemd +using the sd_notify (systemd notifications) protocol and will also tell systemd its current state (Running or Stopped) +when it changes. + +The `freqtrade.service.watchdog` file contains an example of the service unit configuration file which uses systemd +as the watchdog. + +!!! Note + The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container. diff --git a/docs/backtesting.md b/docs/backtesting.md index 474498371..19814303b 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -72,6 +72,8 @@ The exported trades can be used for [further analysis](#further-backtest-result- freqtrade backtesting --export trades --export-filename=backtest_samplestrategy.json ``` +Please also read about the [strategy startup period](strategy-customization.md#strategy-startup-period). + #### Supplying custom fee value Sometimes your account has certain fee rebates (fee reductions starting with a certain account size or monthly volume), which are not visible to ccxt. diff --git a/docs/configuration.md b/docs/configuration.md index 3a8514e73..a11329b05 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -75,8 +75,8 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.key` | '' | API key to use for the exchange. Only required when you are in production mode. ***Keep it in secrete, do not disclose publicly.*** | `exchange.secret` | '' | API secret to use for the exchange. Only required when you are in production mode. ***Keep it in secrete, do not disclose publicly.*** | `exchange.password` | '' | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests. ***Keep it in secrete, do not disclose publicly.*** -| `exchange.pair_whitelist` | [] | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Can be overriden by dynamic pairlists (see [below](#dynamic-pairlists)). -| `exchange.pair_blacklist` | [] | List of pairs the bot must absolutely avoid for trading and backtesting. Can be overriden by dynamic pairlists (see [below](#dynamic-pairlists)). +| `exchange.pair_whitelist` | [] | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used by VolumePairList (see [below](#dynamic-pairlists)). +| `exchange.pair_blacklist` | [] | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#dynamic-pairlists)). | `exchange.ccxt_config` | None | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.ccxt_async_config` | None | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.markets_refresh_interval` | 60 | The interval in minutes in which markets are reloaded. @@ -98,6 +98,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `strategy` | None | **Required** Defines Strategy class to use. Recommended to set via `--strategy NAME`. | `strategy_path` | null | Adds an additional strategy lookup path (must be a directory). | `internals.process_throttle_secs` | 5 | **Required.** Set the process throttle. Value in second. +| `internals.heartbeat_interval` | 60 | Print heartbeat message every X seconds. Set to 0 to disable heartbeat messages. | `internals.sd_notify` | false | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. | `logfile` | | Specify Logfile. Uses a rolling strategy of 10 files, with 1Mb per file. | `user_data_dir` | cwd()/user_data | Directory containing user data. Defaults to `./user_data/`. @@ -134,7 +135,7 @@ To allow the bot to trade all the available `stake_currency` in your account set In this case a trade amount is calclulated as: ```python -currency_balanse / (max_open_trades - current_open_trades) +currency_balance / (max_open_trades - current_open_trades) ``` ### Understand minimal_roi @@ -330,7 +331,7 @@ This configuration enables binance, as well as rate limiting to avoid bans from Optimal settings for rate limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings. We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. -#### Advanced FreqTrade Exchange configuration +#### Advanced Freqtrade Exchange configuration Advanced options can be configured using the `_ft_has_params` setting, which will override Defaults and exchange-specific behaviours. @@ -350,6 +351,13 @@ For example, to test the order type `FOK` with Kraken, and modify candle_limit t !!! Warning Please make sure to fully understand the impacts of these settings before modifying them. +#### Random notes for other exchanges + +* The Ocean (ccxt id: 'theocean') exchange uses Web3 functionality and requires web3 package to be installed: +```shell +$ pip3 install web3 +``` + ### What values can be used for fiat_display_currency? The `fiat_display_currency` configuration parameter sets the base currency to use for the @@ -417,10 +425,16 @@ section of the configuration. `askVolume`, `bidVolume` and `quoteVolume`, defaults to `quoteVolume`. * There is a possibility to filter low-value coins that would not allow setting a stop loss (set `precision_filter` parameter to `true` for this). + * `VolumePairList` does not consider `pair_whitelist`, but builds this automatically based the pairlist configuration. + * Pairs in `pair_blacklist` are not considered for VolumePairList, even if all other filters would match. Example: ```json +"exchange": { + "pair_whitelist": [], + "pair_blacklist": ["BNB/BTC"] +}, "pairlist": { "method": "VolumePairList", "config": { diff --git a/docs/installation.md b/docs/installation.md index 3d0f27f2a..c2e1bb439 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -151,7 +151,7 @@ python3 -m venv .env source .env/bin/activate ``` -#### 3. Install FreqTrade +#### 3. Install Freqtrade Clone the git repository: @@ -192,33 +192,9 @@ freqtrade trade -c config.json *Note*: If you run the bot on a server, you should consider using [Docker](docker.md) or a terminal multiplexer like `screen` or [`tmux`](https://en.wikipedia.org/wiki/Tmux) to avoid that the bot is stopped on logout. -#### 7. [Optional] Configure `freqtrade` as a `systemd` service +#### 7. (Optional) Post-installation Tasks -From the freqtrade repo... copy `freqtrade.service` to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. - -After that you can start the daemon with: - -```bash -systemctl --user start freqtrade -``` - -For this to be persistent (run when user is logged out) you'll need to enable `linger` for your freqtrade user. - -```bash -sudo loginctl enable-linger "$USER" -``` - -If you run the bot as a service, you can use systemd service manager as a software watchdog monitoring freqtrade bot -state and restarting it in the case of failures. If the `internals.sd_notify` parameter is set to true in the -configuration or the `--sd-notify` command line option is used, the bot will send keep-alive ping messages to systemd -using the sd_notify (systemd notifications) protocol and will also tell systemd its current state (Running or Stopped) -when it changes. - -The `freqtrade.service.watchdog` file contains an example of the service unit configuration file which uses systemd -as the watchdog. - -!!! Note - The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container. +On Linux, as an optional post-installation task, you can setup the bot to run as a `systemd` service. See [Advanced Post-installation Tasks](advanced-setup.md) for details. ------ @@ -242,6 +218,12 @@ If that is not available on your system, feel free to try the instructions below ### Install freqtrade manually +!!! Note + Make sure to use 64bit Windows and 64bit Python to avoid problems with backtesting or hyperopt due to the memory constraints 32bit applications have under Windows. + +!!! Hint + Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under Windows can greatly help with installation problems. Check out the [Conda section](#using-conda) in this document. + #### Clone the git repository ```bash diff --git a/docs/stoploss.md b/docs/stoploss.md index f5e2f8df6..105488296 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -3,74 +3,101 @@ The `stoploss` configuration parameter is loss in percentage that should trigger a sale. For example, value `-0.10` will cause immediate sell if the profit dips below -10% for a given trade. This parameter is optional. -Most of the strategy files already include the optimal `stoploss` -value. This parameter is optional. If you use it in the configuration file, it will take over the -`stoploss` value from the strategy file. +Most of the strategy files already include the optimal `stoploss` value. -## Stop Loss support +!!! Info + All stoploss properties mentioned in this file can be set in the Strategy, or in the configuration. Configuration values will override the strategy values. + +## Stop Loss Types At this stage the bot contains the following stoploss support modes: -1. static stop loss, defined in either the strategy or configuration. -2. trailing stop loss, defined in the configuration. -3. trailing stop loss, custom positive loss, defined in configuration. +1. Static stop loss. +2. Trailing stop loss. +3. Trailing stop loss, custom positive loss. +4. Trailing stop loss only once the trade has reached a certain offset. -!!! Note - All stoploss properties can be configured in either Strategy or configuration. Configuration values override strategy values. +Those stoploss modes can be *on exchange* or *off exchange*. If the stoploss is *on exchange* it means a stoploss limit order is placed on the exchange immediately after buy order happens successfully. This will protect you against sudden crashes in market as the order will be in the queue immediately and if market goes down then the order has more chance of being fulfilled. -Those stoploss modes can be *on exchange* or *off exchange*. If the stoploss is *on exchange* it means a stoploss limit order is placed on the exchange immediately after buy order happens successfuly. This will protect you against sudden crashes in market as the order will be in the queue immediately and if market goes down then the order has more chance of being fulfilled. +In case of stoploss on exchange there is another parameter called `stoploss_on_exchange_interval`. This configures the interval in seconds at which the bot will check the stoploss and update it if necessary. -In case of stoploss on exchange there is another parameter called `stoploss_on_exchange_interval`. This configures the interval in seconds at which the bot will check the stoploss and update it if necessary. As an example in case of trailing stoploss if the order is on the exchange and the market is going up then the bot automatically cancels the previous stoploss order and put a new one with a stop value higher than previous one. It is clear that the bot cannot do it every 5 seconds otherwise it gets banned. So this parameter will tell the bot how often it should update the stoploss order. The default value is 60 (1 minute). +For example, assuming the stoploss is on exchange, and trailing stoploss is enabled, and the market is going up, then the bot automatically cancels the previous stoploss order and puts a new one with a stop value higher than the previous stoploss order. +The bot cannot do this every 5 seconds (at each iteration), otherwise it would get banned by the exchange. +So this parameter will tell the bot how often it should update the stoploss order. The default value is 60 (1 minute). +This same logic will reapply a stoploss order on the exchange should you cancel it accidentally. !!! Note Stoploss on exchange is only supported for Binance as of now. ## Static Stop Loss -This is very simple, basically you define a stop loss of x in your strategy file or alternative in the configuration, which -will overwrite the strategy definition. This will basically try to sell your asset, the second the loss exceeds the defined loss. +This is very simple, you define a stop loss of x (as a ratio of price, i.e. x * 100% of price). This will try to sell the asset once the loss exceeds the defined loss. ## Trailing Stop Loss -The initial value for this stop loss, is defined in your strategy or configuration. Just as you would define your Stop Loss normally. -To enable this Feauture all you have to do is to define the configuration element: +The initial value for this is `stoploss`, just as you would define your static Stop loss. +To enable trailing stoploss: -``` json -"trailing_stop" : True +``` python +trailing_stop = True ``` -This will now activate an algorithm, which automatically moves your stop loss up every time the price of your asset increases. +This will now activate an algorithm, which automatically moves the stop loss up every time the price of your asset increases. -For example, simplified math, +For example, simplified math: -* you buy an asset at a price of 100$ -* your stop loss is defined at 2% -* which means your stop loss, gets triggered once your asset dropped below 98$ -* assuming your asset now increases to 102$ -* your stop loss, will now be 2% of 102$ or 99.96$ -* now your asset drops in value to 101$, your stop loss, will still be 99.96$ +* the bot buys an asset at a price of 100$ +* the stop loss is defined at 2% +* the stop loss would get triggered once the asset dropps below 98$ +* assuming the asset now increases to 102$ +* the stop loss will now be 2% of 102$ or 99.96$ +* now the asset drops in value to 101$, the stop loss will still be 99.96$ and would trigger at 99.96$. -basically what this means is that your stop loss will be adjusted to be always be 2% of the highest observed price +In summary: The stoploss will be adjusted to be always be 2% of the highest observed price. -### Custom positive loss +### Custom positive stoploss -Due to demand, it is possible to have a default stop loss, when you are in the red with your buy, but once your profit surpasses a certain percentage, -the system will utilize a new stop loss, which can be a different value. For example your default stop loss is 5%, but once you have 1.1% profit, -it will be changed to be only a 1% stop loss, which trails the green candles until it goes below them. +It is also possible to have a default stop loss, when you are in the red with your buy, but once your profit surpasses a certain percentage, the system will utilize a new stop loss, which can have a different value. +For example your default stop loss is 5%, but once you have 1.1% profit, it will be changed to be only a 1% stop loss, which trails the green candles until it goes below them. -Both values can be configured in the main configuration file and requires `"trailing_stop": true` to be set to true. +Both values require `trailing_stop` to be set to true. -``` json - "trailing_stop_positive": 0.01, - "trailing_stop_positive_offset": 0.011, - "trailing_only_offset_is_reached": false +``` python + trailing_stop_positive = 0.01 + trailing_stop_positive_offset = 0.011 ``` The 0.01 would translate to a 1% stop loss, once you hit 1.1% profit. +Before this, `stoploss` is used for the trailing stoploss. -You should also make sure to have this value (`trailing_stop_positive_offset`) lower than your minimal ROI, otherwise minimal ROI will apply first and sell your trade. +Read the [next section](#trailing-only-once-offset-is-reached) to keep stoploss at 5% of the entry point. -If `"trailing_only_offset_is_reached": true` then the trailing stoploss is only activated once the offset is reached. Until then, the stoploss remains at the configured`stoploss`. +!!! Tip + Make sure to have this value (`trailing_stop_positive_offset`) lower than minimal ROI, otherwise minimal ROI will apply first and sell the trade. + +### Trailing only once offset is reached + +It is also possible to use a static stoploss until the offset is reached, and then trail the trade to take profits once the market turns. + +If `"trailing_only_offset_is_reached": true` then the trailing stoploss is only activated once the offset is reached. Until then, the stoploss remains at the configured `stoploss`. +This option can be used with or without `trailing_stop_positive`, but uses `trailing_stop_positive_offset` as offset. + +``` python + trailing_stop_positive_offset = 0.011 + trailing_only_offset_is_reached = true +``` + +Simplified example: + +``` python + stoploss = 0.05 + trailing_stop_positive_offset = 0.03 + trailing_only_offset_is_reached = True +``` + +* the bot buys an asset at a price of 100$ +* the stop loss is defined at 5% +* the stop loss will remain at 95% until profit reaches +3% ## Changing stoploss on open trades diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index e401d3ab8..808c94174 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -117,6 +117,37 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame Look into the [user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/strategies/sample_strategy.py). Then uncomment indicators you need. +### Strategy startup period + +Most indicators have an instable startup period, in which they are either not available, or the calculation is incorrect. This can lead to inconsistencies, since Freqtrade does not know how long this instable period should be. +To account for this, the strategy can be assigned the `startup_candle_count` attribute. +This should be set to the maximum number of candles that the strategy requires to calculate stable indicators. + +In this example strategy, this should be set to 100 (`startup_candle_count = 100`), since the longest needed history is 100 candles. + +``` python + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) +``` + +By letting the bot know how much history is needed, backtest trades can start at the specified timerange during backtesting and hyperopt. + +!!! Warning + `startup_candle_count` should be below `ohlcv_candle_limit` (which is 500 for most exchanges) - since only this amount of candles will be available during Dry-Run/Live Trade operations. + +#### Example + +Let's try to backtest 1 month (January 2019) of 5m candles using the an example strategy with EMA100, as above. + +``` bash +freqtrade backtesting --timerange 20190101-20190201 --ticker-interval 5m +``` + +Assuming `startup_candle_count` is set to 100, backtesting knows it needs 100 candles to generate valid buy signals. It will load data from `20190101 - (100 * 5m)` - which is ~2019-12-31 15:30:00. +If this data is available, indicators will be calculated with this extended timerange. The instable startup period (up to 2019-01-01 00:00:00) will then be removed before starting backtesting. + +!!! Note + If data for the startup period is not available, then the timerange will be adjusted to account for this startup period - so Backtesting would start at 2019-01-01 08:30:00. + ### Buy signal rules Edit the method `populate_buy_trend()` in your strategy file to update your buy strategy. @@ -267,10 +298,10 @@ class Awesomestrategy(IStrategy): ``` !!! Warning - The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash. + The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash. !!! Note - If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. + If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. ### Additional data (DataProvider) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 6a8374e6d..93d93263f 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -5,7 +5,7 @@ from jsonschema import Draft4Validator, validators from jsonschema.exceptions import ValidationError, best_match from freqtrade import constants, OperationalException - +from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -64,6 +64,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None: # validating trailing stoploss _validate_trailing_stoploss(conf) _validate_edge(conf) + _validate_whitelist(conf) def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None: @@ -111,3 +112,15 @@ def _validate_edge(conf: Dict[str, Any]) -> None: "Edge and VolumePairList are incompatible, " "Edge will override whatever pairs VolumePairlist selects." ) + + +def _validate_whitelist(conf: Dict[str, Any]) -> None: + """ + Dynamic whitelist does not require pair_whitelist to be set - however StaticWhitelist does. + """ + if conf.get('runmode', RunMode.OTHER) in [RunMode.OTHER, RunMode.PLOT]: + return + + if (conf.get('pairlist', {}).get('method', 'StaticPairList') == 'StaticPairList' + and not conf.get('exchange', {}).get('pair_whitelist')): + raise OperationalException("StaticPairList requires pair_whitelist to be set.") diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 393fd78be..b07dbce54 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -183,6 +183,9 @@ class Configuration: config['exchange']['name'] = self.args["exchange"] logger.info(f"Using exchange {config['exchange']['name']}") + if 'pair_whitelist' not in config['exchange']: + config['exchange']['pair_whitelist'] = [] + if 'user_data_dir' in self.args and self.args["user_data_dir"]: config.update({'user_data_dir': self.args["user_data_dir"]}) elif 'user_data_dir' not in config: diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index fc759ab6e..156f0e1e2 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -1,11 +1,14 @@ """ This module contains the argument manager class """ +import logging import re from typing import Optional import arrow +logger = logging.getLogger(__name__) + class TimeRange: """ @@ -27,6 +30,34 @@ class TimeRange: return (self.starttype == other.starttype and self.stoptype == other.stoptype and self.startts == other.startts and self.stopts == other.stopts) + def subtract_start(self, seconds) -> None: + """ + Subtracts from startts if startts is set. + :param seconds: Seconds to subtract from starttime + :return: None (Modifies the object in place) + """ + if self.startts: + self.startts = self.startts - seconds + + def adjust_start_if_necessary(self, ticker_interval_secs: int, startup_candles: int, + min_date: arrow.Arrow) -> None: + """ + Adjust startts by candles. + Applies only if no startup-candles have been available. + :param ticker_interval_secs: Ticker interval in seconds e.g. `timeframe_to_seconds('5m')` + :param startup_candles: Number of candles to move start-date forward + :param min_date: Minimum data date loaded. Key kriterium to decide if start-time + has to be moved + :return: None (Modifies the object in place) + """ + if (not self.starttype or (startup_candles + and min_date.timestamp >= self.startts)): + # If no startts was defined, or backtest-data starts at the defined backtest-date + logger.warning("Moving start-date by %s candles to account for startup time.", + startup_candles) + self.startts = (min_date.timestamp + ticker_interval_secs * startup_candles) + self.starttype = 'date' + @staticmethod def parse_timerange(text: Optional[str]): """ diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 1e9ce7e92..a008c8aaf 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -233,7 +233,7 @@ CONF_SCHEMA = { 'ccxt_config': {'type': 'object'}, 'ccxt_async_config': {'type': 'object'} }, - 'required': ['name', 'pair_whitelist'] + 'required': ['name'] }, 'edge': { 'type': 'object', diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 17abae3b6..0f5d395ff 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -150,15 +150,21 @@ def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame], column: str = "c return df_comb -def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str) -> pd.DataFrame: +def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, + timeframe: str) -> pd.DataFrame: """ Adds a column `col_name` with the cumulative profit for the given trades array. :param df: DataFrame with date index :param trades: DataFrame containing trades (requires columns close_time and profitperc) + :param col_name: Column name that will be assigned the results + :param timeframe: Timeframe used during the operations :return: Returns df with one additional column, col_name, containing the cumulative profit. """ - # Use groupby/sum().cumsum() to avoid errors when multiple trades sold at the same candle. - df[col_name] = trades.groupby('close_time')['profitperc'].sum().cumsum() + from freqtrade.exchange import timeframe_to_minutes + ticker_minutes = timeframe_to_minutes(timeframe) + # Resample to ticker_interval to make sure trades match candles + _trades_sum = trades.resample(f'{ticker_minutes}min', on='close_time')[['profitperc']].sum() + df.loc[:, col_name] = _trades_sum.cumsum() # Set first value to 0 df.loc[df.iloc[0].name, col_name] = 0 # FFill to get continuous diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index ed5d80b0e..412b086c0 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -8,17 +8,19 @@ Includes: import logging import operator +from copy import deepcopy from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Tuple import arrow +import pytz from pandas import DataFrame from freqtrade import OperationalException, misc from freqtrade.configuration import TimeRange from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv -from freqtrade.exchange import Exchange, timeframe_to_minutes +from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_seconds logger = logging.getLogger(__name__) @@ -49,6 +51,19 @@ def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: return tickerlist[start_index:stop_index] +def trim_dataframe(df: DataFrame, timerange: TimeRange) -> DataFrame: + """ + Trim dataframe based on given timerange + """ + if timerange.starttype == 'date': + start = datetime.fromtimestamp(timerange.startts, tz=pytz.utc) + df = df.loc[df['date'] >= start, :] + if timerange.stoptype == 'date': + stop = datetime.fromtimestamp(timerange.stopts, tz=pytz.utc) + df = df.loc[df['date'] <= stop, :] + return df + + def load_tickerdata_file(datadir: Path, pair: str, ticker_interval: str, timerange: Optional[TimeRange] = None) -> Optional[list]: """ @@ -113,7 +128,8 @@ def load_pair_history(pair: str, refresh_pairs: bool = False, exchange: Optional[Exchange] = None, fill_up_missing: bool = True, - drop_incomplete: bool = True + drop_incomplete: bool = True, + startup_candles: int = 0, ) -> DataFrame: """ Loads cached ticker history for the given pair. @@ -126,9 +142,15 @@ def load_pair_history(pair: str, :param exchange: Exchange object (needed when using "refresh_pairs") :param fill_up_missing: Fill missing values with "No action"-candles :param drop_incomplete: Drop last candle assuming it may be incomplete. + :param startup_candles: Additional candles to load at the start of the period :return: DataFrame with ohlcv data """ + timerange_startup = deepcopy(timerange) + if startup_candles > 0 and timerange_startup: + logger.info('Using indicator startup period: %s ...', startup_candles) + timerange_startup.subtract_start(timeframe_to_seconds(ticker_interval) * startup_candles) + # The user forced the refresh of pairs if refresh_pairs: download_pair_history(datadir=datadir, @@ -137,11 +159,11 @@ def load_pair_history(pair: str, ticker_interval=ticker_interval, timerange=timerange) - pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange) + pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange_startup) if pairdata: - if timerange: - _validate_pairdata(pair, pairdata, timerange) + if timerange_startup: + _validate_pairdata(pair, pairdata, timerange_startup) return parse_ticker_dataframe(pairdata, ticker_interval, pair=pair, fill_missing=fill_up_missing, drop_incomplete=drop_incomplete) @@ -160,10 +182,22 @@ def load_data(datadir: Path, exchange: Optional[Exchange] = None, timerange: Optional[TimeRange] = None, fill_up_missing: bool = True, + startup_candles: int = 0, + fail_without_data: bool = False ) -> Dict[str, DataFrame]: """ Loads ticker history data for a list of pairs - :return: dict(:) + :param datadir: Path to the data storage location. + :param ticker_interval: Ticker-interval (e.g. "5m") + :param pairs: List of pairs to load + :param refresh_pairs: Refresh pairs from exchange. + (Note: Requires exchange to be passed as well.) + :param exchange: Exchange object (needed when using "refresh_pairs") + :param timerange: Limit data to be loaded to this timerange + :param fill_up_missing: Fill missing values with "No action"-candles + :param startup_candles: Additional candles to load at the start of the period + :param fail_without_data: Raise OperationalException if no data is found. + :return: dict(:) TODO: refresh_pairs is still used by edge to keep the data uptodate. This should be replaced in the future. Instead, writing the current candles to disk from dataprovider should be implemented, as this would avoid loading ohlcv data twice. @@ -176,9 +210,13 @@ def load_data(datadir: Path, datadir=datadir, timerange=timerange, refresh_pairs=refresh_pairs, exchange=exchange, - fill_up_missing=fill_up_missing) + fill_up_missing=fill_up_missing, + startup_candles=startup_candles) if hist is not None: result[pair] = hist + + if fail_without_data and not result: + raise OperationalException("No data found. Terminating.") return result diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 2655fbc65..883bf4a0f 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -100,7 +100,8 @@ class Edge: ticker_interval=self.strategy.ticker_interval, refresh_pairs=self._refresh_pairs, exchange=self.exchange, - timerange=self._timerange + timerange=self._timerange, + startup_candles=self.strategy.startup_candle_count, ) if not data: diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 71f0737ef..023e16cc5 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -228,6 +228,7 @@ class Exchange: self.validate_pairs(config['exchange']['pair_whitelist']) self.validate_ordertypes(config.get('order_types', {})) self.validate_order_time_in_force(config.get('order_time_in_force', {})) + self.validate_required_startup_candles(config.get('startup_candle_count', 0)) # Converts the interval provided in minutes in config to seconds self.markets_refresh_interval: int = exchange_config.get( @@ -443,6 +444,16 @@ class Exchange: raise OperationalException( f'Time in force policies are not supported for {self.name} yet.') + def validate_required_startup_candles(self, startup_candles) -> None: + """ + Checks if required startup_candles is more than ohlcv_candle_limit. + Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. + """ + if startup_candles + 5 > self._ft_has['ohlcv_candle_limit']: + raise OperationalException( + f"This strategy requires {startup_candles} candles to start. " + f"{self.name} only provides {self._ft_has['ohlcv_candle_limit']}.") + def exchange_has(self, endpoint: str) -> bool: """ Checks if exchange implements a specific API endpoint. diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ed5116b08..a8fc6bc7e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -6,26 +6,27 @@ import logging import traceback from datetime import datetime from math import isclose +from os import getpid from typing import Any, Dict, List, Optional, Tuple import arrow from requests.exceptions import RequestException -from freqtrade import (DependencyException, InvalidOrderException, - __version__, constants, persistence) +from freqtrade import (DependencyException, InvalidOrderException, __version__, + constants, persistence) +from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.configuration import validate_config_consistency from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.persistence import Trade +from freqtrade.resolvers import (ExchangeResolver, PairListResolver, + StrategyResolver) from freqtrade.rpc import RPCManager, RPCMessageType -from freqtrade.resolvers import ExchangeResolver, StrategyResolver, PairListResolver from freqtrade.state import State -from freqtrade.strategy.interface import SellType, IStrategy +from freqtrade.strategy.interface import IStrategy, SellType from freqtrade.wallets import Wallets - logger = logging.getLogger(__name__) @@ -50,13 +51,15 @@ class FreqtradeBot: # Init objects self.config = config + self._heartbeat_msg = 0 + + self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60) + self.strategy: IStrategy = StrategyResolver(self.config).strategy # Check config consistency here since strategies can set certain options validate_config_consistency(config) - self.rpc: RPCManager = RPCManager(self) - self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange self.wallets = Wallets(self.config, self.exchange) @@ -74,7 +77,7 @@ class FreqtradeBot: self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None - self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] + self.active_pair_whitelist = self._refresh_whitelist() persistence.init(self.config.get('db_url', None), clean_open_orders=self.config.get('dry_run', False)) @@ -83,6 +86,13 @@ class FreqtradeBot: initial_state = self.config.get('initial_state') self.state = State[initial_state.upper()] if initial_state else State.STOPPED + # RPC runs in separate threads, can start handling external commands just after + # initialization, even before Freqtradebot has a chance to start its throttling, + # so anything in the Freqtradebot instance should be ready (initialized), including + # the initial state of the bot. + # Keep this at the end of this initialization method. + self.rpc: RPCManager = RPCManager(self) + def cleanup(self) -> None: """ Cleanup pending resources on an already stopped bot @@ -113,21 +123,10 @@ class FreqtradeBot: # Check whether markets have to be reloaded self.exchange._reload_markets() - # Refresh whitelist - self.pairlists.refresh_pairlist() - self.active_pair_whitelist = self.pairlists.whitelist - - # Calculating Edge positioning - if self.edge: - self.edge.calculate() - self.active_pair_whitelist = self.edge.adjust(self.active_pair_whitelist) - # Query trades from persistence layer trades = Trade.get_open_trades() - # Extend active-pair whitelist with pairs from open trades - # It ensures that tickers are downloaded for open trades - self._extend_whitelist_with_trades(self.active_pair_whitelist, trades) + self.active_pair_whitelist = self._refresh_whitelist(trades) # Refreshing candles self.dataprovider.refresh(self._create_pair_whitelist(self.active_pair_whitelist), @@ -145,11 +144,29 @@ class FreqtradeBot: self.check_handle_timedout() Trade.session.flush() - def _extend_whitelist_with_trades(self, whitelist: List[str], trades: List[Any]): + if (self.heartbeat_interval + and (arrow.utcnow().timestamp - self._heartbeat_msg > self.heartbeat_interval)): + logger.info(f"Bot heartbeat. PID={getpid()}") + self._heartbeat_msg = arrow.utcnow().timestamp + + def _refresh_whitelist(self, trades: List[Trade] = []) -> List[str]: """ - Extend whitelist with pairs from open trades + Refresh whitelist from pairlist or edge and extend it with trades. """ - whitelist.extend([trade.pair for trade in trades if trade.pair not in whitelist]) + # Refresh whitelist + self.pairlists.refresh_pairlist() + _whitelist = self.pairlists.whitelist + + # Calculating Edge positioning + if self.edge: + self.edge.calculate() + _whitelist = self.edge.adjust(_whitelist) + + if trades: + # Extend active-pair whitelist with pairs from open trades + # It ensures that tickers are downloaded for open trades + _whitelist.extend([trade.pair for trade in trades if trade.pair not in _whitelist]) + return _whitelist def _create_pair_whitelist(self, pairs: List[str]) -> List[Tuple[str, str]]: """ @@ -438,7 +455,7 @@ class FreqtradeBot: try: # Create entity and execute trade if not self.create_trades(): - logger.info('Found no buy signals for whitelisted currencies. Trying again...') + logger.debug('Found no buy signals for whitelisted currencies. Trying again...') except DependencyException as exception: logger.warning('Unable to create trade: %s', exception) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fb8c182ee..fe31912bc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -15,7 +15,7 @@ from freqtrade import OperationalException from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.dataprovider import DataProvider -from freqtrade.exchange import timeframe_to_minutes +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -90,6 +90,8 @@ class Backtesting: self.ticker_interval = str(self.config.get('ticker_interval')) self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval) + # Get maximum required startup period + self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) # Load one (first) strategy self._set_strategy(self.strategylist[0]) @@ -103,6 +105,31 @@ class Backtesting: # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False + def load_bt_data(self): + timerange = TimeRange.parse_timerange(None if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) + + data = history.load_data( + datadir=Path(self.config['datadir']), + pairs=self.config['exchange']['pair_whitelist'], + ticker_interval=self.ticker_interval, + timerange=timerange, + startup_candles=self.required_startup, + fail_without_data=True, + ) + + min_date, max_date = history.get_timeframe(data) + + logger.info( + 'Loading data from %s up to %s (%s days)..', + min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days + ) + # Adjust startts forward if not enough data is available + timerange.adjust_start_if_necessary(timeframe_to_seconds(self.ticker_interval), + self.required_startup, min_date) + + return data, timerange + def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame, skip_nan: bool = False) -> str: """ @@ -412,39 +439,18 @@ class Backtesting: :return: None """ data: Dict[str, Any] = {} - pairs = self.config['exchange']['pair_whitelist'] logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - - timerange = TimeRange.parse_timerange(None if self.config.get( - 'timerange') is None else str(self.config.get('timerange'))) - data = history.load_data( - datadir=Path(self.config['datadir']), - pairs=pairs, - ticker_interval=self.ticker_interval, - timerange=timerange, - ) - - if not data: - logger.critical("No data found. Terminating.") - return # Use max_open_trades in backtesting, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): max_open_trades = self.config['max_open_trades'] else: logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 + + data, timerange = self.load_bt_data() + all_results = {} - - min_date, max_date = history.get_timeframe(data) - - logger.info( - 'Backtesting with data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days - ) - for strat in self.strategylist: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) self._set_strategy(strat) @@ -452,6 +458,15 @@ class Backtesting: # need to reprocess data every time to populate signals preprocessed = self.strategy.tickerdata_to_dataframe(data) + # Trim startup period from analyzed dataframe + for pair, df in preprocessed.items(): + preprocessed[pair] = history.trim_dataframe(df, timerange) + min_date, max_date = history.get_timeframe(preprocessed) + + logger.info( + 'Backtesting with data from %s up to %s (%s days)..', + min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days + ) # Execute backtest and print results all_results[self.strategy.get_strategy_name()] = self.backtest( { diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 07258a048..c576ea6f8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,8 +22,7 @@ from pandas import DataFrame from skopt import Optimizer from skopt.space import Dimension -from freqtrade.configuration import TimeRange -from freqtrade.data.history import load_data, get_timeframe +from freqtrade.data.history import get_timeframe, trim_dataframe from freqtrade.misc import round_dict from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules @@ -379,30 +378,19 @@ class Hyperopt: ) def start(self) -> None: - timerange = TimeRange.parse_timerange(None if self.config.get( - 'timerange') is None else str(self.config.get('timerange'))) - data = load_data( - datadir=Path(self.config['datadir']), - pairs=self.config['exchange']['pair_whitelist'], - ticker_interval=self.backtesting.ticker_interval, - timerange=timerange - ) + data, timerange = self.backtesting.load_bt_data() - if not data: - logger.critical("No data found. Terminating.") - return + preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data) + # Trim startup period from analyzed dataframe + for pair, df in preprocessed.items(): + preprocessed[pair] = trim_dataframe(df, timerange) min_date, max_date = get_timeframe(data) logger.info( 'Hyperopting with data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days + min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days ) - - preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data) - dump(preprocessed, self.tickerdata_pickle) # We don't need exchange instance anymore while running hyperopt diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index b9b7977ab..5f53cd17b 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -54,7 +54,7 @@ class VolumePairList(IPairList): """ # Generate dynamic whitelist self._whitelist = self._gen_pair_whitelist( - self._config['stake_currency'], self._sort_key)[:self._number_pairs] + self._config['stake_currency'], self._sort_key) @cached(TTLCache(maxsize=1, ttl=1800)) def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: @@ -91,6 +91,6 @@ class VolumePairList(IPairList): valid_tickers.remove(t) pairs = [s['symbol'] for s in valid_tickers] - logger.info(f"Searching pairs: {self._whitelist}") + logger.info(f"Searching pairs: {pairs[:self._number_pairs]}") return pairs diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 6bd5993b6..bbdb52ca1 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -264,12 +264,12 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], - trades: pd.DataFrame) -> go.Figure: + trades: pd.DataFrame, timeframe: str) -> go.Figure: # Combine close-values for all pairs, rename columns to "pair" df_comb = combine_tickers_with_mean(tickers, "close") # Add combined cumulative profit - df_comb = create_cum_profit(df_comb, trades, 'cum_profit') + df_comb = create_cum_profit(df_comb, trades, 'cum_profit', timeframe) # Plot the pairs average close prices, and total profit growth avgclose = go.Scatter( @@ -293,7 +293,7 @@ def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], for pair in pairs: profit_col = f'cum_profit_{pair}' - df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col) + df_comb = create_cum_profit(df_comb, trades[trades['pair'] == pair], profit_col, timeframe) fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}") @@ -382,9 +382,9 @@ def plot_profit(config: Dict[str, Any]) -> None: ) # Filter trades to relevant pairs trades = trades[trades['pair'].isin(plot_elements["pairs"])] - # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend - fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], trades) + fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], + trades, config.get('ticker_interval', '5m')) store_plot_file(fig, filename='freqtrade-profit-plot.html', directory=config['user_data_dir'] / "plot", auto_open=True) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 22a26a5f3..102816981 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -60,6 +60,7 @@ class StrategyResolver(IResolver): ("order_time_in_force", None, False), ("stake_currency", None, False), ("stake_amount", None, False), + ("startup_candle_count", None, False), ("use_sell_signal", True, True), ("sell_profit_only", False, True), ("ignore_roi_if_buy_signal", False, True), diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index b839a9618..6c343b477 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -39,6 +39,9 @@ class DefaultStrategy(IStrategy): 'stoploss_on_exchange': False } + # Number of candles the strategy requires before producing valid signals + startup_candle_count: int = 20 + # Optional time in force for orders order_time_in_force = { 'buy': 'gtc', @@ -105,9 +108,6 @@ class DefaultStrategy(IStrategy): # EMA - Exponential Moving Average dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - return dataframe def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 014ca9968..d42f8e989 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -103,6 +103,9 @@ class IStrategy(ABC): # run "populate_indicators" only for new candle process_only_new_candles: bool = False + # Count of candles the strategy requires before producing valid signals + startup_candle_count: int = 0 + # Class level variables (intentional) containing # the dataprovider (dp) (access to other candles, historic data, ...) # and wallets - access to the current balance. @@ -421,6 +424,7 @@ class IStrategy(ABC): def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: """ Creates a dataframe and populates indicators for given ticker data + Used by optimize operations only, not during dry / live runs. """ return {pair: self.advise_indicators(pair_data, {'pair': pair}) for pair, pair_data in tickerdata.items()} diff --git a/freqtrade/utils.py b/freqtrade/utils.py index 506ed0a31..692dc8ba0 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -103,9 +103,9 @@ def start_download_data(args: Dict[str, Any]) -> None: pairs_not_available: List[str] = [] + # Init exchange + exchange = ExchangeResolver(config['exchange']['name'], config).exchange try: - # Init exchange - exchange = ExchangeResolver(config['exchange']['name'], config).exchange if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( @@ -127,7 +127,7 @@ def start_download_data(args: Dict[str, Any]) -> None: finally: if pairs_not_available: logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {config['exchange']['name']}.") + f"on exchange {exchange.name}.") def start_list_timeframes(args: Dict[str, Any]) -> None: @@ -144,8 +144,8 @@ def start_list_timeframes(args: Dict[str, Any]) -> None: if args['print_one_column']: print('\n'.join(exchange.timeframes)) else: - print(f"Timeframes available for the exchange `{config['exchange']['name']}`: " - f"{', '.join(exchange.timeframes)}.") + print(f"Timeframes available for the exchange `{exchange.name}`: " + f"{', '.join(exchange.timeframes)}") def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: diff --git a/mkdocs.yml b/mkdocs.yml index 863731873..2c3f70191 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,6 +22,7 @@ nav: - Strategy analysis: strategy_analysis_example.md - Plotting: plotting.md - SQL Cheatsheet: sql_cheatsheet.md + - Advanced Post-installation Tasks: advanced-setup.md - Sandbox Testing: sandbox-testing.md - Deprecated Features: deprecated.md - Contributors Guide: developer.md diff --git a/requirements-common.txt b/requirements-common.txt index 1e42d8a04..64a43ee62 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,6 +1,6 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.18.1306 +ccxt==1.18.1346 SQLAlchemy==1.3.10 python-telegram-bot==12.2.0 arrow==0.15.2 diff --git a/requirements-dev.txt b/requirements-dev.txt index f5cde59e8..589ca7c54 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,11 +8,11 @@ flake8==3.7.8 flake8-type-annotations==0.1.0 flake8-tidy-imports==3.0.0 mypy==0.740 -pytest==5.2.1 +pytest==5.2.2 pytest-asyncio==0.10.0 pytest-cov==2.8.1 -pytest-mock==1.11.1 +pytest-mock==1.11.2 pytest-random-order==1.0.4 # Convert jupyter notebooks to markdown documents -nbconvert==5.6.0 +nbconvert==5.6.1 diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py deleted file mode 100755 index a8f919a10..000000000 --- a/scripts/download_backtest_data.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -import sys - - -print("This script has been integrated into freqtrade " - "and its functionality is available by calling `freqtrade download-data`.") -print("Please check the documentation on https://www.freqtrade.io/en/latest/backtesting/ " - "for details.") - -sys.exit(1) diff --git a/scripts/get_market_pairs.py b/scripts/get_market_pairs.py deleted file mode 100644 index cd38bf2fa..000000000 --- a/scripts/get_market_pairs.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -This script was adapted from ccxt here: -https://github.com/ccxt/ccxt/blob/master/examples/py/arbitrage-pairs.py -""" -import os -import sys -import traceback - -root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -sys.path.append(root + '/python') - -import ccxt # noqa: E402 - - -def style(s, style): - return style + s + '\033[0m' - - -def green(s): - return style(s, '\033[92m') - - -def blue(s): - return style(s, '\033[94m') - - -def yellow(s): - return style(s, '\033[93m') - - -def red(s): - return style(s, '\033[91m') - - -def pink(s): - return style(s, '\033[95m') - - -def bold(s): - return style(s, '\033[1m') - - -def underline(s): - return style(s, '\033[4m') - - -def dump(*args): - print(' '.join([str(arg) for arg in args])) - - -def print_supported_exchanges(): - dump('Supported exchanges:', green(', '.join(ccxt.exchanges))) - - -try: - - if len(sys.argv) < 2: - dump("Usage: python " + sys.argv[0], green('id')) - print_supported_exchanges() - sys.exit(1) - - id = sys.argv[1] # get exchange id from command line arguments - - # check if the exchange is supported by ccxt - exchange_found = id in ccxt.exchanges - - if exchange_found: - dump('Instantiating', green(id), 'exchange') - - # instantiate the exchange by id - exchange = getattr(ccxt, id)({ - # 'proxy':'https://cors-anywhere.herokuapp.com/', - }) - - # load all markets from the exchange - markets = exchange.load_markets() - - # output a list of all market symbols - dump(green(id), 'has', len(exchange.symbols), 'symbols:', exchange.symbols) - - tuples = list(ccxt.Exchange.keysort(markets).items()) - - # debug - for (k, v) in tuples: - print(v) - - # output a table of all markets - dump(pink('{:<15} {:<15} {:<15} {:<15}'.format('id', 'symbol', 'base', 'quote'))) - - for (k, v) in tuples: - dump('{:<15} {:<15} {:<15} {:<15}'.format(v['id'], v['symbol'], v['base'], v['quote'])) - - else: - - dump('Exchange ' + red(id) + ' not found') - print_supported_exchanges() - -except Exception as e: - dump('[' + type(e).__name__ + ']', str(e)) - dump(traceback.format_exc()) - dump("Usage: python " + sys.argv[0], green('id')) - print_supported_exchanges() - sys.exit(1) diff --git a/scripts/plot_dataframe.py b/scripts/plot_dataframe.py deleted file mode 100755 index 62c4bc39f..000000000 --- a/scripts/plot_dataframe.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -import sys - - -print("This script has been integrated into freqtrade " - "and its functionality is available by calling `freqtrade plot-dataframe`.") -print("Please check the documentation on https://www.freqtrade.io/en/latest/plotting/ " - "for details.") - -sys.exit(1) diff --git a/scripts/plot_profit.py b/scripts/plot_profit.py deleted file mode 100755 index c9a23c1ee..000000000 --- a/scripts/plot_profit.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -import sys - - -print("This script has been integrated into freqtrade " - "and its functionality is available by calling `freqtrade plot-profit`.") -print("Please check the documentation on https://www.freqtrade.io/en/latest/plotting/ " - "for details.") - -sys.exit(1) diff --git a/tests/conftest.py b/tests/conftest.py index 4946c4d42..c5cfd702b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,13 +55,16 @@ def patched_configuration_load_config_file(mocker, config) -> None: ) -def patch_exchange(mocker, api_mock=None, id='bittrex') -> None: +def patch_exchange(mocker, api_mock=None, id='bittrex', mock_markets=True) -> None: mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.validate_ordertypes', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value=id)) mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value=id.title())) + if mock_markets: + mocker.patch('freqtrade.exchange.Exchange.markets', + PropertyMock(return_value=get_markets())) if api_mock: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) @@ -69,8 +72,9 @@ def patch_exchange(mocker, api_mock=None, id='bittrex') -> None: mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) -def get_patched_exchange(mocker, config, api_mock=None, id='bittrex') -> Exchange: - patch_exchange(mocker, api_mock, id) +def get_patched_exchange(mocker, config, api_mock=None, id='bittrex', + mock_markets=True) -> Exchange: + patch_exchange(mocker, api_mock, id, mock_markets) config["exchange"]["name"] = id try: exchange = ExchangeResolver(id, config).exchange @@ -85,6 +89,11 @@ def patch_wallet(mocker, free=999.9) -> None: )) +def patch_whitelist(mocker, conf) -> None: + mocker.patch('freqtrade.freqtradebot.FreqtradeBot._refresh_whitelist', + MagicMock(return_value=conf['exchange']['pair_whitelist'])) + + def patch_edge(mocker) -> None: # "ETH/BTC", # "LTC/BTC", @@ -120,6 +129,7 @@ def patch_freqtradebot(mocker, config) -> None: patch_exchange(mocker) mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) + patch_whitelist(mocker, config) def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: @@ -288,6 +298,10 @@ def ticker_sell_down(): @pytest.fixture def markets(): + return get_markets() + + +def get_markets(): return { 'ETH/BTC': { 'id': 'ethbtc', @@ -370,7 +384,7 @@ def markets(): 'symbol': 'LTC/BTC', 'base': 'LTC', 'quote': 'BTC', - 'active': False, + 'active': True, 'precision': { 'price': 8, 'amount': 8, @@ -395,7 +409,7 @@ def markets(): 'symbol': 'XRP/BTC', 'base': 'XRP', 'quote': 'BTC', - 'active': False, + 'active': True, 'precision': { 'price': 8, 'amount': 8, @@ -420,7 +434,7 @@ def markets(): 'symbol': 'NEO/BTC', 'base': 'NEO', 'quote': 'BTC', - 'active': False, + 'active': True, 'precision': { 'price': 8, 'amount': 8, @@ -445,7 +459,7 @@ def markets(): 'symbol': 'BTT/BTC', 'base': 'BTT', 'quote': 'BTC', - 'active': True, + 'active': False, 'precision': { 'base': 8, 'quote': 8, @@ -495,7 +509,7 @@ def markets(): 'symbol': 'LTC/USDT', 'base': 'LTC', 'quote': 'USDT', - 'active': True, + 'active': False, 'precision': { 'amount': 8, 'price': 8 diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 4068e00e4..a04a2c529 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock import pytest from arrow import Arrow -from pandas import DataFrame, to_datetime +from pandas import DataFrame, DateOffset, to_datetime from freqtrade.configuration import TimeRange from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, @@ -130,7 +130,25 @@ def test_create_cum_profit(testdatadir): cum_profits = create_cum_profit(df.set_index('date'), bt_data[bt_data["pair"] == 'POWR/BTC'], - "cum_profits") + "cum_profits", timeframe="5m") + assert "cum_profits" in cum_profits.columns + assert cum_profits.iloc[0]['cum_profits'] == 0 + assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005 + + +def test_create_cum_profit1(testdatadir): + filename = testdatadir / "backtest-result_test.json" + bt_data = load_backtest_data(filename) + # Move close-time to "off" the candle, to make sure the logic still works + bt_data.loc[:, 'close_time'] = bt_data.loc[:, 'close_time'] + DateOffset(seconds=20) + timerange = TimeRange.parse_timerange("20180110-20180112") + + df = load_pair_history(pair="POWR/BTC", ticker_interval='5m', + datadir=testdatadir, timerange=timerange) + + cum_profits = create_cum_profit(df.set_index('date'), + bt_data[bt_data["pair"] == 'POWR/BTC'], + "cum_profits", timeframe="5m") assert "cum_profits" in cum_profits.columns assert cum_profits.iloc[0]['cum_profits'] == 0 assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005 diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 95382768a..d9627a0e4 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -95,6 +95,23 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> N _clean_test_file(file) +def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None: + ltfmock = mocker.patch('freqtrade.data.history.load_tickerdata_file', + MagicMock(return_value=None)) + timerange = TimeRange('date', None, 1510639620, 0) + history.load_pair_history(pair='UNITTEST/BTC', ticker_interval='1m', + datadir=testdatadir, timerange=timerange, + startup_candles=20, + ) + assert log_has( + 'Using indicator startup period: 20 ...', caplog + ) + assert ltfmock.call_count == 1 + assert ltfmock.call_args_list[0][1]['timerange'] != timerange + # startts is 20 minutes earlier + assert ltfmock.call_args_list[0][1]['timerange'].startts == timerange.startts - 20 * 60 + + def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, default_conf, testdatadir) -> None: """ @@ -427,6 +444,46 @@ def test_trim_tickerlist(testdatadir) -> None: assert not ticker +def test_trim_dataframe(testdatadir) -> None: + data = history.load_data( + datadir=testdatadir, + ticker_interval='1m', + pairs=['UNITTEST/BTC'] + )['UNITTEST/BTC'] + min_date = int(data.iloc[0]['date'].timestamp()) + max_date = int(data.iloc[-1]['date'].timestamp()) + data_modify = data.copy() + + # Remove first 30 minutes (1800 s) + tr = TimeRange('date', None, min_date + 1800, 0) + data_modify = history.trim_dataframe(data_modify, tr) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 30 + assert all(data_modify.iloc[-1] == data.iloc[-1]) + assert all(data_modify.iloc[0] == data.iloc[30]) + + data_modify = data.copy() + # Remove last 30 minutes (1800 s) + tr = TimeRange(None, 'date', 0, max_date - 1800) + data_modify = history.trim_dataframe(data_modify, tr) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 30 + assert all(data_modify.iloc[0] == data.iloc[0]) + assert all(data_modify.iloc[-1] == data.iloc[-31]) + + data_modify = data.copy() + # Remove first 25 and last 30 minutes (1800 s) + tr = TimeRange('date', 'date', min_date + 1500, max_date - 1800) + data_modify = history.trim_dataframe(data_modify, tr) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 55 + # first row matches 25th original row + assert all(data_modify.iloc[0] == data.iloc[25]) + + def test_file_dump_json_tofile(testdatadir) -> None: file = testdatadir / 'test_{id}.json'.format(id=str(uuid.uuid4())) data = {'bar': 'foo'} @@ -533,21 +590,22 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): dl_mock = mocker.patch('freqtrade.data.history.download_pair_history', MagicMock()) + + ex = get_patched_exchange(mocker, default_conf) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) ) - ex = get_patched_exchange(mocker, default_conf) timerange = TimeRange.parse_timerange("20190101-20190102") - unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], + unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["BTT/BTC", "LTC/USDT"], timeframes=["1m", "5m"], dl_path=testdatadir, timerange=timerange, erase=False ) assert dl_mock.call_count == 0 - assert "ETH/BTC" in unav_pairs - assert "XRP/BTC" in unav_pairs - assert log_has("Skipping pair ETH/BTC...", caplog) + assert "BTT/BTC" in unav_pairs + assert "LTC/USDT" in unav_pairs + assert log_has("Skipping pair BTT/BTC...", caplog) def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir): diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index 5e244a97e..e1af50768 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -256,7 +256,7 @@ def test_edge_heartbeat_calculate(mocker, edge_conf): def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, - timerange=None, exchange=None): + timerange=None, exchange=None, *args, **kwargs): hz = 0.1 base = 0.001 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 1e0a5fdc3..4eb0df1a3 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -177,16 +177,11 @@ def test_symbol_amount_prec(default_conf, mocker): ''' Test rounds down to 4 Decimal places ''' - api_mock = MagicMock() - api_mock.load_markets = MagicMock(return_value={ - 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' - }) - mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='binance')) markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'amount': 4}}}) - type(api_mock).markets = markets - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange = get_patched_exchange(mocker, default_conf, id="binance") + mocker.patch('freqtrade.exchange.Exchange.markets', markets) amount = 2.34559 pair = 'ETH/BTC' @@ -198,16 +193,10 @@ def test_symbol_price_prec(default_conf, mocker): ''' Test rounds up to 4 decimal places ''' - api_mock = MagicMock() - api_mock.load_markets = MagicMock(return_value={ - 'ETH/BTC': '', 'LTC/BTC': '', 'XRP/BTC': '', 'NEO/BTC': '' - }) - mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='binance')) - markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'price': 4}}}) - type(api_mock).markets = markets - exchange = get_patched_exchange(mocker, default_conf, api_mock) + exchange = get_patched_exchange(mocker, default_conf, id="binance") + mocker.patch('freqtrade.exchange.Exchange.markets', markets) price = 2.34559 pair = 'ETH/BTC' @@ -279,7 +268,7 @@ def test__load_markets(default_conf, mocker, caplog): api_mock.load_markets = MagicMock(return_value=expected_return) type(api_mock).markets = expected_return default_conf['exchange']['pair_whitelist'] = ['ETH/BTC'] - ex = get_patched_exchange(mocker, default_conf, api_mock, id="binance") + ex = get_patched_exchange(mocker, default_conf, api_mock, id="binance", mock_markets=False) assert ex.markets == expected_return @@ -294,7 +283,8 @@ def test__reload_markets(default_conf, mocker, caplog): api_mock.load_markets = load_markets type(api_mock).markets = initial_markets default_conf['exchange']['markets_refresh_interval'] = 10 - exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance") + exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance", + mock_markets=False) exchange._last_markets_refresh = arrow.utcnow().timestamp updated_markets = {'ETH/BTC': {}, "LTC/BTC": {}} @@ -533,6 +523,24 @@ def test_validate_order_types_not_in_config(default_conf, mocker): Exchange(conf) +def test_validate_required_startup_candles(default_conf, mocker, caplog): + api_mock = MagicMock() + mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) + + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + + default_conf['startup_candle_count'] = 20 + ex = Exchange(default_conf) + assert ex + default_conf['startup_candle_count'] = 600 + + with pytest.raises(OperationalException, match=r'This strategy requires 600.*'): + Exchange(default_conf) + + def test_exchange_has(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) assert not exchange.exchange_has('ASDFASDF') @@ -1715,15 +1723,16 @@ def test_get_valid_pair_combination(default_conf, mocker, markets): 'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC']), # active markets ([], [], False, True, - ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/USD', 'LTC/USDT', - 'TKN/BTC', 'XLTCUSDT']), + ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'NEO/BTC', + 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC']), # all pairs ([], [], True, False, ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XRP/BTC']), # active pairs ([], [], True, True, - ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/USD', 'LTC/USDT', 'TKN/BTC']), + ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'NEO/BTC', + 'TKN/BTC', 'XRP/BTC']), # all markets, base=ETH, LTC (['ETH', 'LTC'], [], False, False, ['ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 285b73e21..dea2d42f0 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -117,7 +117,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, - timerange=None, exchange=None, live=False): + timerange=None, exchange=None, live=False, *args, **kwargs): tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC", fill_missing=True)} @@ -494,7 +494,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> def get_timeframe(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={})) + mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=None)) mocker.patch('freqtrade.data.history.get_timeframe', get_timeframe) mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) @@ -511,10 +511,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> default_conf['timerange'] = '20180101-20180102' backtesting = Backtesting(default_conf) - backtesting.start() - # check the logs, that will contain the backtest result - - assert log_has('No data found. Terminating.', caplog) + with pytest.raises(OperationalException, match='No data found. Terminating.'): + backtesting.start() def test_backtest(default_conf, fee, mocker, testdatadir) -> None: @@ -838,6 +836,8 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', + 'Loading data from 2017-11-14T20:57:00+00:00 ' + 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...' @@ -892,6 +892,8 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', + 'Loading data from 2017-11-14T20:57:00+00:00 ' + 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 6bed0f8cc..1d5b59886 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -244,7 +244,7 @@ def test_start(mocker, default_conf, caplog) -> None: def test_start_no_data(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock(return_value={})) + mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=None)) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -259,9 +259,8 @@ def test_start_no_data(mocker, default_conf, caplog) -> None: '--epochs', '5' ] args = get_args(args) - start_hyperopt(args) - - assert log_has('No data found. Terminating.', caplog) + with pytest.raises(OperationalException, match='No data found. Terminating.'): + start_hyperopt(args) def test_start_filelock(mocker, default_conf, caplog) -> None: @@ -411,7 +410,8 @@ def test_roi_table_generation(hyperopt) -> None: def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -631,7 +631,8 @@ def test_continue_hyperopt(mocker, default_conf, caplog): def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -669,7 +670,8 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -707,7 +709,8 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -754,7 +757,8 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -> None: mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -784,7 +788,8 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) - def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -830,7 +835,8 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) @@ -882,7 +888,8 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None ]) def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, method, space) -> None: mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', + MagicMock(return_value=(MagicMock(), None))) mocker.patch( 'freqtrade.optimize.hyperopt.get_timeframe', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 411ae60a3..929fc0ba0 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -80,7 +80,7 @@ def test_refresh_pairlist_dynamic(mocker, markets, tickers, whitelist_conf): freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) # argument: use the whitelist dynamically by exchange-volume - whitelist = ['ETH/BTC', 'TKN/BTC', 'BTT/BTC'] + whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'] freqtradebot.pairlists.refresh_pairlist() assert whitelist == freqtradebot.pairlists.whitelist @@ -108,12 +108,12 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): @pytest.mark.parametrize("precision_filter,base_currency,key,whitelist_result", [ - (False, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'BTT/BTC']), - (False, "BTC", "bidVolume", ['BTT/BTC', 'TKN/BTC', 'ETH/BTC']), - (False, "USDT", "quoteVolume", ['ETH/USDT', 'LTC/USDT']), + (False, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), + (False, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC']), + (False, "USDT", "quoteVolume", ['ETH/USDT']), (False, "ETH", "quoteVolume", []), # this replaces tests that were removed from test_exchange - (True, "BTC", "quoteVolume", ["ETH/BTC", "TKN/BTC"]), - (True, "BTC", "bidVolume", ["TKN/BTC", "ETH/BTC"]) + (True, "BTC", "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC"]), + (True, "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC"]) ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers, base_currency, key, whitelist_result, precision_filter) -> None: @@ -127,7 +127,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers, freqtrade.pairlists._precision_filter = precision_filter freqtrade.config['stake_currency'] = base_currency whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency=base_currency, key=key) - assert whitelist == whitelist_result + assert sorted(whitelist) == sorted(whitelist_result) def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: @@ -160,7 +160,7 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"), # TRX/ETH wrong stake (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"), # BCH/BTC not available (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "is not compatible with exchange"), # BLK/BTC in blacklist - (['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], "Market is not active") # LTC/BTC is inactive + (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active") # BTT/BTC is inactive ]) def test_validate_whitelist(mocker, whitelist_conf, markets, pairlist, whitelist, caplog, log_message): diff --git a/scripts/pytest.sh b/tests/pytest.sh similarity index 100% rename from scripts/pytest.sh rename to tests/pytest.sh diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 66468927f..df2261c1f 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -9,12 +9,11 @@ from numpy import isnan from freqtrade import DependencyException, TemporaryError from freqtrade.edge import PairInfo -from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State -from tests.conftest import patch_exchange, patch_get_signal +from tests.conftest import patch_get_signal, get_patched_freqtradebot # Functions for recurrent object patching @@ -26,17 +25,15 @@ def prec_satoshi(a, b) -> float: # Unit tests -def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: +def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) @@ -98,17 +95,15 @@ def test_rpc_trade_status(default_conf, ticker, fee, markets, mocker) -> None: } == results[0] -def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: - patch_exchange(mocker) +def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) @@ -134,7 +129,6 @@ def test_rpc_status_table(default_conf, ticker, fee, markets, mocker) -> None: def test_rpc_daily_profit(default_conf, update, ticker, fee, limit_buy_order, limit_sell_order, markets, mocker) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -143,7 +137,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -181,22 +175,20 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, - limit_buy_order, limit_sell_order, markets, mocker) -> None: + limit_buy_order, limit_sell_order, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) - patch_exchange(mocker) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -267,9 +259,8 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Test that rpc_trade_statistics can handle trades that lacks # trade.open_rate (it is set to None) -def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, +def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, ticker_sell_up, limit_buy_order, limit_sell_order): - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.rpc.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), @@ -281,10 +272,9 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, markets, 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) stake_currency = default_conf['stake_currency'] fiat_display_currency = default_conf['fiat_display_currency'] @@ -343,7 +333,6 @@ def test_rpc_balance_handle_error(default_conf, mocker): 'freqtrade.rpc.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) - patch_exchange(mocker) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -352,7 +341,7 @@ def test_rpc_balance_handle_error(default_conf, mocker): get_ticker=MagicMock(side_effect=TemporaryError('Could not load ticker due to xxx')) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) rpc._fiat_converter = CryptoToFiatConverter() @@ -394,7 +383,6 @@ def test_rpc_balance_handle(default_conf, mocker): 'freqtrade.rpc.fiat_convert.Market', ticker=MagicMock(return_value={'price_usd': 15000.0}), ) - patch_exchange(mocker) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( @@ -406,7 +394,7 @@ def test_rpc_balance_handle(default_conf, mocker): side_effect=lambda a, b: f"{b}/{a}" if a == "PAX" else f"{a}/{b}") ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) rpc._fiat_converter = CryptoToFiatConverter() @@ -438,14 +426,13 @@ def test_rpc_balance_handle(default_conf, mocker): def test_rpc_start(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=MagicMock() ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.STOPPED @@ -460,14 +447,13 @@ def test_rpc_start(mocker, default_conf) -> None: def test_rpc_stop(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=MagicMock() ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.RUNNING @@ -483,14 +469,13 @@ def test_rpc_stop(mocker, default_conf) -> None: def test_rpc_stopbuy(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=MagicMock() ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) freqtradebot.state = State.RUNNING @@ -501,8 +486,7 @@ def test_rpc_stopbuy(mocker, default_conf) -> None: assert freqtradebot.config['max_open_trades'] == 0 -def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: - patch_exchange(mocker) +def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) cancel_order_mock = MagicMock() @@ -518,10 +502,9 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: } ), get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) @@ -606,18 +589,16 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker, markets) -> None: def test_performance_handle(default_conf, ticker, limit_buy_order, fee, - limit_sell_order, markets, mocker) -> None: - patch_exchange(mocker) + limit_sell_order, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) @@ -641,18 +622,16 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, assert prec_satoshi(res[0]['profit'], 6.2) -def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: - patch_exchange(mocker) +def test_rpc_count(mocker, default_conf, ticker, fee) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) @@ -665,9 +644,8 @@ def test_rpc_count(mocker, default_conf, ticker, fee, markets) -> None: assert counts["current"] == 1 -def test_rpcforcebuy(mocker, default_conf, ticker, fee, markets, limit_buy_order) -> None: +def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None: default_conf['forcebuy_enable'] = True - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( @@ -675,11 +653,10 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, markets, limit_buy_order get_balances=MagicMock(return_value=ticker), get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets), buy=buy_mm ) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) pair = 'ETH/BTC' @@ -704,7 +681,7 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, markets, limit_buy_order # Test not buying default_conf['stake_amount'] = 0.0000001 - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) pair = 'TKN/BTC' @@ -715,10 +692,9 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, markets, limit_buy_order def test_rpcforcebuy_stopped(mocker, default_conf) -> None: default_conf['forcebuy_enable'] = True default_conf['initial_state'] = 'stopped' - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) pair = 'ETH/BTC' @@ -727,10 +703,9 @@ def test_rpcforcebuy_stopped(mocker, default_conf) -> None: def test_rpcforcebuy_disabled(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) pair = 'ETH/BTC' @@ -739,10 +714,9 @@ def test_rpcforcebuy_disabled(mocker, default_conf) -> None: def test_rpc_whitelist(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_whitelist() assert ret['method'] == 'StaticPairList' @@ -750,14 +724,13 @@ def test_rpc_whitelist(mocker, default_conf) -> None: def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: - patch_exchange(mocker) default_conf['pairlist'] = {'method': 'VolumePairList', 'config': {'number_assets': 4} } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_whitelist() assert ret['method'] == 'VolumePairList' @@ -766,10 +739,9 @@ def test_rpc_whitelist_dynamic(mocker, default_conf) -> None: def test_rpc_blacklist(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_blacklist(None) assert ret['method'] == 'StaticPairList' @@ -785,23 +757,21 @@ def test_rpc_blacklist(mocker, default_conf) -> None: def test_rpc_edge_disabled(mocker, default_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtradebot) with pytest.raises(RPCException, match=r'Edge is not enabled.'): rpc._rpc_edge() def test_rpc_edge_enabled(mocker, edge_conf) -> None: - patch_exchange(mocker) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': PairInfo(-0.02, 0.66, 3.71, 0.50, 1.71, 10, 60), } )) - freqtradebot = FreqtradeBot(edge_conf) + freqtradebot = get_patched_freqtradebot(mocker, edge_conf) rpc = RPC(freqtradebot) ret = rpc._rpc_edge() diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index a776ad5df..766511d2d 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -22,7 +22,7 @@ from freqtrade.rpc.telegram import Telegram, authorized_only from freqtrade.state import State from freqtrade.strategy.interface import SellType from tests.conftest import (get_patched_freqtradebot, log_has, patch_exchange, - patch_get_signal) + patch_get_signal, patch_whitelist) class DummyCls(Telegram): @@ -143,17 +143,15 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: assert log_has('Exception occurred within Telegram module', caplog) -def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: +def test_status(default_conf, update, mocker, fee, ticker,) -> None: update.message.chat.id = 123 default_conf['telegram']['enabled'] = False default_conf['telegram']['chat_id'] = 123 - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(markets) ) msg_mock = MagicMock() status_table = MagicMock() @@ -184,9 +182,8 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: _status_table=status_table, _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -204,13 +201,11 @@ def test_status(default_conf, update, mocker, fee, ticker, markets) -> None: assert status_table.call_count == 1 -def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> None: - patch_exchange(mocker) +def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(markets) ) msg_mock = MagicMock() status_table = MagicMock() @@ -220,9 +215,9 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No _status_table=status_table, _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -256,14 +251,12 @@ def test_status_handle(default_conf, update, ticker, fee, markets, mocker) -> No assert 'ETH/BTC' in msg_mock.call_args_list[0][0][0] -def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) -> None: - patch_exchange(mocker) +def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, - markets=PropertyMock(markets) ) msg_mock = MagicMock() mocker.patch.multiple( @@ -271,10 +264,9 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) default_conf['stake_amount'] = 15.0 - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -307,8 +299,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, markets, mocker) def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, - limit_sell_order, markets, mocker) -> None: - patch_exchange(mocker) + limit_sell_order, mocker) -> None: default_conf['max_open_trades'] = 1 mocker.patch( 'freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', @@ -318,7 +309,6 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(markets) ) msg_mock = MagicMock() mocker.patch.multiple( @@ -326,9 +316,8 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -382,7 +371,6 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker @@ -393,9 +381,8 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -420,14 +407,12 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, - limit_buy_order, limit_sell_order, markets, mocker) -> None: - patch_exchange(mocker) + limit_buy_order, limit_sell_order, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(markets) ) msg_mock = MagicMock() mocker.patch.multiple( @@ -435,9 +420,8 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, _init=MagicMock(), _send_msg=msg_mock ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -724,16 +708,16 @@ def test_reload_conf_handle(default_conf, update, mocker) -> None: def test_forcesell_handle(default_conf, update, ticker, fee, - ticker_sell_up, markets, mocker) -> None: + ticker_sell_up, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) patch_exchange(mocker) + patch_whitelist(mocker, default_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets), ) freqtradebot = FreqtradeBot(default_conf) @@ -775,17 +759,18 @@ def test_forcesell_handle(default_conf, update, ticker, fee, def test_forcesell_down_handle(default_conf, update, ticker, fee, - ticker_sell_down, markets, mocker) -> None: + ticker_sell_down, mocker) -> None: mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) patch_exchange(mocker) + patch_whitelist(mocker, default_conf) + mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets), ) freqtradebot = FreqtradeBot(default_conf) @@ -830,17 +815,17 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, } == last_msg -def test_forcesell_all_handle(default_conf, update, ticker, fee, markets, mocker) -> None: +def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None: patch_exchange(mocker) mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + patch_whitelist(mocker, default_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets), ) default_conf['max_open_trades'] = 4 freqtradebot = FreqtradeBot(default_conf) @@ -885,9 +870,8 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: _init=MagicMock(), _send_msg=msg_mock ) - patch_exchange(mocker) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -980,8 +964,7 @@ def test_forcebuy_handle_exception(default_conf, update, markets, mocker) -> Non def test_performance_handle(default_conf, update, ticker, fee, - limit_buy_order, limit_sell_order, markets, mocker) -> None: - patch_exchange(mocker) + limit_buy_order, limit_sell_order, mocker) -> None: msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -992,10 +975,8 @@ def test_performance_handle(default_conf, update, ticker, fee, 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(markets), ) - mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) @@ -1018,8 +999,7 @@ def test_performance_handle(default_conf, update, ticker, fee, assert 'ETH/BTC\t6.20% (1)' in msg_mock.call_args_list[0][0][0] -def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> None: - patch_exchange(mocker) +def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', @@ -1030,10 +1010,9 @@ def test_count_handle(default_conf, update, ticker, fee, markets, mocker) -> Non 'freqtrade.exchange.Exchange', get_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}), - markets=PropertyMock(markets) + get_fee=fee, ) - mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - freqtradebot = FreqtradeBot(default_conf) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index e43040958..697cb8b48 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -751,6 +751,30 @@ def test_validate_edge(edge_conf): validate_config_consistency(edge_conf) +def test_validate_whitelist(default_conf): + default_conf['runmode'] = RunMode.DRY_RUN + # Test regular case - has whitelist and uses StaticPairlist + validate_config_consistency(default_conf) + conf = deepcopy(default_conf) + del conf['exchange']['pair_whitelist'] + # Test error case + with pytest.raises(OperationalException, + match="StaticPairList requires pair_whitelist to be set."): + + validate_config_consistency(conf) + + conf = deepcopy(default_conf) + + conf.update({"pairlist": { + "method": "VolumePairList", + }}) + # Dynamic whitelist should not care about pair_whitelist + validate_config_consistency(conf) + del conf['exchange']['pair_whitelist'] + + validate_config_consistency(conf) + + def test_load_config_test_comments() -> None: """ Load config with comments @@ -823,7 +847,7 @@ def test_pairlist_resolving(): args = Arguments(arglist).get_parsed_arg() - configuration = Configuration(args) + configuration = Configuration(args, RunMode.OTHER) config = configuration.get_config() assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] @@ -917,7 +941,7 @@ def test_pairlist_resolving_fallback(mocker): # Fix flaky tests if config.json exists args["config"] = None - configuration = Configuration(args) + configuration = Configuration(args, RunMode.OTHER) config = configuration.get_config() assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index f1533d867..30f9ba0a4 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -23,7 +23,7 @@ from freqtrade.strategy.interface import SellCheckTuple, SellType from freqtrade.worker import Worker from tests.conftest import (get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange, - patch_get_signal, patch_wallet) + patch_get_signal, patch_wallet, patch_whitelist) def patch_RPCManager(mocker) -> MagicMock: @@ -1247,11 +1247,10 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, - markets, limit_buy_order, limit_sell_order) -> None: + limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set stoploss_limit = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=MagicMock(return_value={ @@ -1262,7 +1261,6 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), stoploss_limit=stoploss_limit ) @@ -1272,7 +1270,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, # disabling ROI default_conf['minimal_roi']['0'] = 999999999 - freqtrade = FreqtradeBot(default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # enabling stoploss on exchange freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -1521,6 +1519,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, def test_process_maybe_execute_buys(mocker, default_conf, caplog) -> None: + caplog.set_level(logging.DEBUG) freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trades', MagicMock(return_value=False)) @@ -1824,20 +1823,18 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, def test_handle_trade_roi(default_conf, ticker, limit_buy_order, - fee, mocker, markets, caplog) -> None: + fee, mocker, caplog) -> None: caplog.set_level(logging.DEBUG) patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtrade, value=(True, False)) freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) @@ -1858,20 +1855,18 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, def test_handle_trade_use_sell_signal( - default_conf, ticker, limit_buy_order, fee, mocker, markets, caplog) -> None: + default_conf, ticker, limit_buy_order, fee, mocker, caplog) -> None: # use_sell_signal is True buy default caplog.set_level(logging.DEBUG) patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) - freqtrade = FreqtradeBot(default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) freqtrade.create_trades() @@ -2236,6 +2231,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, markets, moc get_fee=fee, markets=PropertyMock(return_value=markets) ) + patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2282,6 +2278,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, markets, get_fee=fee, markets=PropertyMock(return_value=markets) ) + patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2331,6 +2328,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe get_fee=fee, markets=PropertyMock(return_value=markets) ) + patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2647,6 +2645,7 @@ def test_execute_sell_market_order(default_conf, ticker, fee, get_fee=fee, markets=PropertyMock(return_value=markets) ) + patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2814,14 +2813,13 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, marke assert trade.sell_reason == SellType.SELL_SIGNAL.value -def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, markets, mocker, caplog) -> None: +def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2851,7 +2849,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, markets, mock assert log_has(f"Pair {trade.pair} is currently locked.", caplog) -def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, mocker) -> None: +def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2863,7 +2861,6 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets) ) default_conf['ask_strategy'] = { 'ignore_roi_if_buy_signal': True @@ -2885,7 +2882,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, markets, m assert trade.sell_reason == SellType.ROI.value -def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, mocker) -> None: +def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -2897,9 +2894,9 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), ) default_conf['trailing_stop'] = True + patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) @@ -2937,7 +2934,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, markets, caplog, assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value -def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets, +def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, caplog, mocker) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) @@ -2951,10 +2948,11 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), ) default_conf['trailing_stop'] = True default_conf['trailing_stop_positive'] = 0.01 + patch_whitelist(mocker, default_conf) + freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) @@ -2994,7 +2992,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, - caplog, mocker, markets) -> None: + caplog, mocker) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) patch_exchange(mocker) @@ -3007,9 +3005,8 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), ) - + patch_whitelist(mocker, default_conf) default_conf['trailing_stop'] = True default_conf['trailing_stop_positive'] = 0.01 default_conf['trailing_stop_positive_offset'] = 0.011 @@ -3054,7 +3051,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, - caplog, mocker, markets) -> None: + caplog, mocker) -> None: buy_price = limit_buy_order['price'] # buy_price: 0.00001099 @@ -3069,9 +3066,8 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, }), buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, - markets=PropertyMock(return_value=markets), ) - + patch_whitelist(mocker, default_conf) default_conf['trailing_stop'] = True default_conf['trailing_stop_positive'] = 0.05 default_conf['trailing_stop_positive_offset'] = 0.055 @@ -3647,3 +3643,27 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker): ftbot = get_patched_freqtradebot(mocker, edge_conf) ftbot.startup() assert reinit_mock.call_count == 0 + + +def test_process_i_am_alive(default_conf, mocker, caplog): + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + ftbot = get_patched_freqtradebot(mocker, default_conf) + message = r"Bot heartbeat\. PID=.*" + ftbot.process() + assert log_has_re(message, caplog) + assert ftbot._heartbeat_msg != 0 + + caplog.clear() + # Message is not shown before interval is up + ftbot.process() + assert not log_has_re(message, caplog) + + caplog.clear() + # Set clock - 70 seconds + ftbot._heartbeat_msg -= 70 + + ftbot.process() + assert log_has_re(message, caplog) diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 7699ae8bc..9e932b6f5 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -234,7 +234,7 @@ def test_add_profit(testdatadir): cum_profits = create_cum_profit(df.set_index('date'), bt_data[bt_data["pair"] == 'POWR/BTC'], - "cum_profits") + "cum_profits", timeframe="5m") fig1 = add_profit(fig, row=2, data=cum_profits, column='cum_profits', name='Profits') figure = fig1.layout.figure @@ -256,7 +256,7 @@ def test_generate_profit_graph(testdatadir): ) trades = trades[trades['pair'].isin(pairs)] - fig = generate_profit_graph(pairs, tickers, trades) + fig = generate_profit_graph(pairs, tickers, trades, timeframe="5m") assert isinstance(fig, go.Figure) assert fig.layout.title.text == "Freqtrade Profit plot" diff --git a/tests/test_timerange.py b/tests/test_timerange.py index 4851cbebd..5c35535f0 100644 --- a/tests/test_timerange.py +++ b/tests/test_timerange.py @@ -1,10 +1,11 @@ # pragma pylint: disable=missing-docstring, C0103 +import arrow import pytest from freqtrade.configuration import TimeRange -def test_parse_timerange_incorrect() -> None: +def test_parse_timerange_incorrect(): assert TimeRange('date', None, 1274486400, 0) == TimeRange.parse_timerange('20100522-') assert TimeRange(None, 'date', 0, 1274486400) == TimeRange.parse_timerange('-20100522') @@ -28,3 +29,37 @@ def test_parse_timerange_incorrect() -> None: with pytest.raises(Exception, match=r'Incorrect syntax.*'): TimeRange.parse_timerange('-') + + +def test_subtract_start(): + x = TimeRange('date', 'date', 1274486400, 1438214400) + x.subtract_start(300) + assert x.startts == 1274486400 - 300 + + # Do nothing if no startdate exists + x = TimeRange(None, 'date', 0, 1438214400) + x.subtract_start(300) + assert not x.startts + + x = TimeRange('date', None, 1274486400, 0) + x.subtract_start(300) + assert x.startts == 1274486400 - 300 + + +def test_adjust_start_if_necessary(): + min_date = arrow.Arrow(2017, 11, 14, 21, 15, 00) + + x = TimeRange('date', 'date', 1510694100, 1510780500) + # Adjust by 20 candles - min_date == startts + x.adjust_start_if_necessary(300, 20, min_date) + assert x.startts == 1510694100 + (20 * 300) + + x = TimeRange('date', 'date', 1510700100, 1510780500) + # Do nothing, startup is set and different min_date + x.adjust_start_if_necessary(300, 20, min_date) + assert x.startts == 1510694100 + (20 * 300) + + x = TimeRange(None, 'date', 0, 1510780500) + # Adjust by 20 candles = 20 * 5m + x.adjust_start_if_necessary(300, 20, min_date) + assert x.startts == 1510694100 + (20 * 300) diff --git a/tests/test_utils.py b/tests/test_utils.py index 8cd9c3aef..30367858a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -100,7 +100,7 @@ def test_list_timeframes(mocker, capsys): ] start_list_timeframes(get_args(args)) captured = capsys.readouterr() - assert re.match("Timeframes available for the exchange `bittrex`: " + assert re.match("Timeframes available for the exchange `Bittrex`: " "1m, 5m, 30m, 1h, 1d", captured.out) @@ -111,7 +111,7 @@ def test_list_timeframes(mocker, capsys): ] start_list_timeframes(get_args(args)) captured = capsys.readouterr() - assert re.match("Timeframes available for the exchange `bittrex`: " + assert re.match("Timeframes available for the exchange `Bittrex`: " "1m, 5m, 30m, 1h, 1d", captured.out) @@ -125,7 +125,7 @@ def test_list_timeframes(mocker, capsys): '1d': '1d', '3d': '3d', } - patch_exchange(mocker, api_mock=api_mock) + patch_exchange(mocker, api_mock=api_mock, id='binance') # Test with --exchange binance args = [ "list-timeframes", @@ -133,7 +133,7 @@ def test_list_timeframes(mocker, capsys): ] start_list_timeframes(get_args(args)) captured = capsys.readouterr() - assert re.match("Timeframes available for the exchange `binance`: " + assert re.match("Timeframes available for the exchange `Binance`: " "1m, 5m, 15m, 30m, 1h, 6h, 12h, 1d, 3d", captured.out) @@ -188,8 +188,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 8 active markets: " - "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/USD, LTC/USDT, TKN/BTC, XLTCUSDT.\n" + assert ("Exchange Bittrex has 9 active markets: " + "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, NEO/BTC, TKN/BTC, XLTCUSDT, XRP/BTC.\n" in captured.out) patch_exchange(mocker, api_mock=api_mock, id="binance") @@ -202,7 +202,7 @@ def test_list_markets(mocker, markets, capsys): pargs['config'] = None start_list_markets(pargs, False) captured = capsys.readouterr() - assert re.match("\nExchange Binance has 8 active markets:\n", + assert re.match("\nExchange Binance has 9 active markets:\n", captured.out) patch_exchange(mocker, api_mock=api_mock, id="bittrex") @@ -227,8 +227,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), True) captured = capsys.readouterr() - assert ("Exchange Bittrex has 7 active pairs: " - "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/USD, LTC/USDT, TKN/BTC.\n" + assert ("Exchange Bittrex has 8 active pairs: " + "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, NEO/BTC, TKN/BTC, XRP/BTC.\n" in captured.out) # Test list-pairs subcommand with --all: all pairs @@ -254,7 +254,7 @@ def test_list_markets(mocker, markets, capsys): start_list_markets(get_args(args), False) captured = capsys.readouterr() assert ("Exchange Bittrex has 5 active markets with ETH, LTC as base currencies: " - "ETH/BTC, ETH/USDT, LTC/USD, LTC/USDT, XLTCUSDT.\n" + "ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, base=LTC @@ -267,7 +267,7 @@ def test_list_markets(mocker, markets, capsys): start_list_markets(get_args(args), False) captured = capsys.readouterr() assert ("Exchange Bittrex has 3 active markets with LTC as base currency: " - "LTC/USD, LTC/USDT, XLTCUSDT.\n" + "LTC/BTC, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, quote=USDT, USD @@ -279,8 +279,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 4 active markets with USDT, USD as quote currencies: " - "ETH/USDT, LTC/USD, LTC/USDT, XLTCUSDT.\n" + assert ("Exchange Bittrex has 3 active markets with USDT, USD as quote currencies: " + "ETH/USDT, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, quote=USDT @@ -292,8 +292,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 3 active markets with USDT as quote currency: " - "ETH/USDT, LTC/USDT, XLTCUSDT.\n" + assert ("Exchange Bittrex has 2 active markets with USDT as quote currency: " + "ETH/USDT, XLTCUSDT.\n" in captured.out) # active markets, base=LTC, quote=USDT @@ -305,21 +305,21 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 2 active markets with LTC as base currency and " - "with USDT as quote currency: LTC/USDT, XLTCUSDT.\n" + assert ("Exchange Bittrex has 1 active market with LTC as base currency and " + "with USDT as quote currency: XLTCUSDT.\n" in captured.out) # active pairs, base=LTC, quote=USDT args = [ "list-pairs", '--config', 'config.json.example', - "--base", "LTC", "--quote", "USDT", + "--base", "LTC", "--quote", "USD", "--print-list", ] start_list_markets(get_args(args), True) captured = capsys.readouterr() assert ("Exchange Bittrex has 1 active pair with LTC as base currency and " - "with USDT as quote currency: LTC/USDT.\n" + "with USD as quote currency: LTC/USD.\n" in captured.out) # active markets, base=LTC, quote=USDT, NONEXISTENT @@ -331,8 +331,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 2 active markets with LTC as base currency and " - "with USDT, NONEXISTENT as quote currencies: LTC/USDT, XLTCUSDT.\n" + assert ("Exchange Bittrex has 1 active market with LTC as base currency and " + "with USDT, NONEXISTENT as quote currencies: XLTCUSDT.\n" in captured.out) # active markets, base=LTC, quote=NONEXISTENT @@ -355,7 +355,7 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 8 active markets:\n" + assert ("Exchange Bittrex has 9 active markets:\n" in captured.out) # Test tabular output, no markets found @@ -378,7 +378,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ('["BLK/BTC","BTT/BTC","ETH/BTC","ETH/USDT","LTC/USD","LTC/USDT","TKN/BTC","XLTCUSDT"]' + assert ('["BLK/BTC","ETH/BTC","ETH/USDT","LTC/BTC","LTC/USD","NEO/BTC",' + '"TKN/BTC","XLTCUSDT","XRP/BTC"]' in captured.out) # Test --print-csv @@ -391,7 +392,7 @@ def test_list_markets(mocker, markets, capsys): captured = capsys.readouterr() assert ("Id,Symbol,Base,Quote,Active,Is pair" in captured.out) assert ("blkbtc,BLK/BTC,BLK,BTC,True,True" in captured.out) - assert ("BTTBTC,BTT/BTC,BTT,BTC,True,True" in captured.out) + assert ("USD-LTC,LTC/USD,LTC,USD,True,True" in captured.out) # Test --one-column args = [ @@ -402,7 +403,7 @@ def test_list_markets(mocker, markets, capsys): start_list_markets(get_args(args), False) captured = capsys.readouterr() assert re.search(r"^BLK/BTC$", captured.out, re.MULTILINE) - assert re.search(r"^BTT/BTC$", captured.out, re.MULTILINE) + assert re.search(r"^LTC/USD$", captured.out, re.MULTILINE) def test_create_datadir_failed(caplog): @@ -449,7 +450,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): def test_download_data_no_markets(mocker, caplog): dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) - patch_exchange(mocker) + patch_exchange(mocker, id='binance') mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}) ) @@ -461,7 +462,7 @@ def test_download_data_no_markets(mocker, caplog): ] start_download_data(get_args(args)) assert dl_mock.call_args[1]['timerange'].starttype == "date" - assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange binance.", caplog) + assert log_has("Pairs [ETH/BTC,XRP/BTC] not available on exchange Binance.", caplog) def test_download_data_no_exchange(mocker, caplog): diff --git a/user_data/strategies/sample_strategy.py b/user_data/strategies/sample_strategy.py index 80c30283d..c2fd681d2 100644 --- a/user_data/strategies/sample_strategy.py +++ b/user_data/strategies/sample_strategy.py @@ -59,6 +59,9 @@ class SampleStrategy(IStrategy): sell_profit_only = False ignore_roi_if_buy_signal = False + # Number of candles the strategy requires before producing valid signals + startup_candle_count: int = 20 + # Optional order type mapping. order_types = { 'buy': 'limit',