diff --git a/config.json.example b/config.json.example index d6806c0e9..bbd9648da 100644 --- a/config.json.example +++ b/config.json.example @@ -53,6 +53,21 @@ "sell_profit_only": false, "ignore_roi_if_buy_signal": false }, + "edge": { + "enabled": false, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 7, + "total_capital_in_stake_currency": 0.5, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "minimum_winrate": 0.60, + "minimum_expectancy": 0.20, + "min_trade_number": 10, + "max_trade_duration_minute": 1440, + "remove_pumps": false + }, "telegram": { "enabled": true, "token": "your_telegram_token", diff --git a/config_full.json.example b/config_full.json.example index af6c7c045..9dba8f539 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -59,6 +59,20 @@ ], "outdated_offset": 5 }, + "edge": { + "enabled": false, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 2, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "minimum_winrate": 0.60, + "minimum_expectancy": 0.20, + "min_trade_number": 10, + "max_trade_duration_minute": 1440, + "remove_pumps": false + }, "experimental": { "use_sell_signal": false, "sell_profit_only": false, diff --git a/docs/configuration.md b/docs/configuration.md index 15ba4b48d..d70a47b38 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -13,7 +13,7 @@ This page explains how to configure your `config.json` file. We recommend to copy and use the `config.json.example` as a template for your bot configuration. -The table below will list all configuration parameters. +The table below will list all configuration parameters. | Command | Default | Mandatory | Description | |----------|---------|----------|-------------| @@ -21,11 +21,11 @@ The table below will list all configuration parameters. | `stake_currency` | BTC | Yes | Crypto-currency used for trading. | `stake_amount` | 0.05 | Yes | Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to 'unlimited' to allow the bot to use all avaliable balance. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes -| `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. +| `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. -| `process_only_new_candles` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. -| `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. -| `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. +| `process_only_new_candles` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. +| `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. +| `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stop` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). | `trailing_stop_positve` | 0 | No | Changes stop-loss once profit has been reached. | `trailing_stop_positve_offset` | 0 | No | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. @@ -47,6 +47,7 @@ The table below will list all configuration parameters. | `exchange.ccxt_rate_limit` | True | No | DEPRECATED!! Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. | `exchange.ccxt_config` | None | No | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.ccxt_async_config` | None | No | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) +| `edge` | false | No | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` @@ -70,7 +71,7 @@ The definition of each config parameters is in [misc.py](https://github.com/freq ### Understand stake_amount `stake_amount` is an amount of crypto-currency your bot will use for each trade. -The minimal value is 0.0005. If there is not enough crypto-currency in +The minimal value is 0.0005. If there is not enough crypto-currency in the account an exception is generated. To allow the bot to trade all the avaliable `stake_currency` in your account set `stake_amount` = `unlimited`. In this case a trade amount is calclulated as `currency_balanse / (max_open_trades - current_open_trades)`. @@ -186,13 +187,13 @@ creating trades. } ``` -Once you will be happy with your bot performance, you can switch it to +Once you will be happy with your bot performance, you can switch it to production mode. ## Switch to production mode -In production mode, the bot will engage your money. Be careful a wrong -strategy can lose all your money. Be aware of what you are doing when +In production mode, the bot will engage your money. Be careful a wrong +strategy can lose all your money. Be aware of what you are doing when you run it in production mode. ### To switch your bot in production mode: @@ -242,7 +243,7 @@ freqtrade ### Embedding Strategies -FreqTrade provides you with with an easy way to embed the strategy into your configuration file. +FreqTrade provides you with with an easy way to embed the strategy into your configuration file. This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, in your chosen config file. diff --git a/docs/edge.md b/docs/edge.md new file mode 100644 index 000000000..f74f8fdcc --- /dev/null +++ b/docs/edge.md @@ -0,0 +1,151 @@ +# Edge positioning + +This page explains how to use Edge Positioning module in your bot in order to enter into a trade only if the trade has a reasonable win rate and risk reward ratio, and consequently adjust your position size and stoploss. + +**NOTICE:** Edge positioning is not compatible with dynamic whitelist. it overrides dynamic whitelist. + +## Table of Contents + +- [Introduction](#introduction) +- [How does it work?](#how-does-it-work?) +- [Configurations](#configurations) + +## Introduction +Trading is all about probability. No one can claim that he has a strategy working all the time. You have to assume that sometimes you lose.

+But it doesn't mean there is no rule, it only means rules should work "most of the time". Let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. Is it an interesting game ? no, it is quite boring, isn't it?

+But let's say the probability that we have heads is 80%, and the probability that we have tails is 20%. Now it is becoming interesting ... +That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. That means over time you will win 8$ risking only 2$ on each toss of coin.

+Let's complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. The calculation is: 80% * 2$ versus 20% * 8$. It is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

+The question is: How do you calculate that? how do you know if you wanna play? +The answer comes to two factors: +- Win Rate +- Risk Reward Ratio + + +### Win Rate +Means over X trades what is the percentage of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). + + +`W = (Number of winning trades) / (Number of losing trades)` + +### Risk Reward Ratio +Risk Reward Ratio is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose: + +`R = Profit / Loss` + +Over time, on many trades, you can calculate your risk reward by dividing your average profit on winning trades by your average loss on losing trades: + +`Average profit = (Sum of profits) / (Number of winning trades)` + +`Average loss = (Sum of losses) / (Number of losing trades)` + +`R = (Average profit) / (Average loss)` + +### Expectancy + +At this point we can combine W and R to create an expectancy ratio. This is a simple process of multiplying the risk reward ratio by the percentage of winning trades, and subtracting the percentage of losing trades, which is calculated as follows: + +Expectancy Ratio = (Risk Reward Ratio x Win Rate) – Loss Rate + +So lets say your Win rate is 28% and your Risk Reward Ratio is 5: + +`Expectancy = (5 * 0.28) - 0.72 = 0.68` + +Superficially, this means that on average you expect this strategy’s trades to return .68 times the size of your losers. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. + +It is important to remember that any system with an expectancy greater than 0 is profitable using past data. The key is finding one that will be profitable in the future. + +You can also use this number to evaluate the effectiveness of modifications to this system. + +**NOTICE:** It's important to keep in mind that Edge is testing your expectancy using historical data , there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology, but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades. + +## How does it work? +If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. It then calculates win rate and expectancy over X trades for each stoploss. Here is an example: + +| Pair | Stoploss | Win Rate | Risk Reward Ratio | Expectancy | +|----------|:-------------:|-------------:|------------------:|-----------:| +| XZC/ETH | -0.03 | 0.52 |1.359670 | 0.228 | +| XZC/ETH | -0.01 | 0.50 |1.176384 | 0.088 | +| XZC/ETH | -0.02 | 0.51 |1.115941 | 0.079 | + +The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. In the above example stoploss at 3% leads to the maximum expectancy according to historical data. + +Edge then forces stoploss to your strategy dynamically. + +### Position size +Edge dictates the stake amount for each trade to the bot according to the following factors: + +- Allowed capital at risk +- Stoploss + +Allowed capital at risk is calculated as follows: + +**allowed capital at risk** = **total capital** X **allowed risk per trade** + +**total capital** is your stake amount. + +**Stoploss** is calculated as described above against historical data. + +Your position size then will be: + +**position size** = **allowed capital at risk** / **stoploss** + +Example: +Let's say your stake amount is 3 ETH, you would allow 1% of risk for each trade. thus your allowed capital at risk would be **3 x 0.01 = 0.03 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.03 / 0.02= 1.5ETH**.
+ +## Configurations +Edge has following configurations: + +#### enabled +If true, then Edge will run periodically
+(default to false) + +#### process_throttle_secs +How often should Edge run in seconds?
+(default to 3600 so one hour) + +#### calculate_since_number_of_days +Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy +Note that it downloads historical data so increasing this number would lead to slowing down the bot
+(default to 7) + +#### allowed_risk +Percentage of allowed risk per trade
+(default to 0.01 [1%]) + +#### stoploss_range_min +Minimum stoploss
+(default to -0.01) + +#### stoploss_range_max +Maximum stoploss
+(default to -0.10) + +#### stoploss_range_step +As an example if this is set to -0.01 then Edge will test the strategy for [-0.01, -0,02, -0,03 ..., -0.09, -0.10] ranges. +Note than having a smaller step means having a bigger range which could lead to slow calculation.
+if you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10.
+(default to -0.01) + +#### minimum_winrate +It filters pairs which don't have at least minimum_winrate. +This comes handy if you want to be conservative and don't comprise win rate in favor of risk reward ratio.
+(default to 0.60) + +#### minimum_expectancy +It filters paris which have an expectancy lower than this number . +Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return.
+(default to 0.20) + +#### min_trade_number +When calculating W and R and E (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable. Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
+(default to 10, it is highly recommended not to decrease this number) + +#### max_trade_duration_minute +Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
+**NOTICE:** While configuring this value, you should take into consideration your ticker interval. as an example filtering out trades having duration less than one day for a strategy which has 4h interval does not make sense. default value is set assuming your strategy interval is relatively small (1m or 5m, etc).
+(default to 1 day, 1440 = 60 * 24) + +#### remove_pumps +Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
+(default to false) diff --git a/docs/index.md b/docs/index.md index 730f1095e..e6e643ba7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,8 +1,8 @@ # freqtrade documentation Welcome to freqtrade documentation. Please feel free to contribute to -this documentation if you see it became outdated by sending us a -Pull-request. Do not hesitate to reach us on +this documentation if you see it became outdated by sending us a +Pull-request. Do not hesitate to reach us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) if you do not find the answer to your questions. @@ -25,6 +25,7 @@ Pull-request. Do not hesitate to reach us on - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Edge positioning](https://github.com/mishaker/freqtrade/blob/money_mgt/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 2b09aa6c9..b7c069c45 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -37,7 +37,7 @@ SUPPORTED_FIAT = [ "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD", "BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT" - ] +] # Required json-schema for user specified config CONF_SCHEMA = { @@ -102,6 +102,7 @@ CONF_SCHEMA = { } }, 'exchange': {'$ref': '#/definitions/exchange'}, + 'edge': {'$ref': '#/definitions/edge'}, 'experimental': { 'type': 'object', 'properties': { @@ -170,6 +171,23 @@ CONF_SCHEMA = { 'ccxt_async_config': {'type': 'object'} }, 'required': ['name', 'key', 'secret', 'pair_whitelist'] + }, + 'edge': { + 'type': 'object', + 'properties': { + "enabled": {'type': 'boolean'}, + "process_throttle_secs": {'type': 'integer', 'minimum': 600}, + "calculate_since_number_of_days": {'type': 'integer'}, + "allowed_risk": {'type': 'number'}, + "stoploss_range_min": {'type': 'number'}, + "stoploss_range_max": {'type': 'number'}, + "stoploss_range_step": {'type': 'number'}, + "minimum_winrate": {'type': 'number'}, + "minimum_expectancy": {'type': 'number'}, + "min_trade_number": {'type': 'number'}, + "max_trade_duration_minute": {'type': 'integer'}, + "remove_pumps": {'type': 'boolean'} + } } }, 'anyOf': [ diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py new file mode 100644 index 000000000..7b25a8306 --- /dev/null +++ b/freqtrade/edge/__init__.py @@ -0,0 +1,408 @@ +# pragma pylint: disable=W0603 +""" Edge positioning package """ +import logging +from typing import Any, Dict +from collections import namedtuple +import arrow + +import numpy as np +import utils_find_1st as utf1st +from pandas import DataFrame + +import freqtrade.optimize as optimize +from freqtrade.arguments import Arguments +from freqtrade.arguments import TimeRange +from freqtrade.strategy.interface import SellType + + +logger = logging.getLogger(__name__) + + +class Edge(): + """ + Calculates Win Rate, Risk Reward Ratio, Expectancy + against historical data for a give set of markets and a strategy + it then adjusts stoploss and position size accordingly + and force it into the strategy + Author: https://github.com/mishaker + """ + + config: Dict = {} + _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + + # pair info data type + _pair_info = namedtuple( + 'pair_info', + ['stoploss', 'winrate', 'risk_reward_ratio', 'required_risk_reward', 'expectancy']) + + def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: + + self.config = config + self.exchange = exchange + self.strategy = strategy + self.ticker_interval = self.strategy.ticker_interval + self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe + self.get_timeframe = optimize.get_timeframe + self.advise_sell = self.strategy.advise_sell + self.advise_buy = self.strategy.advise_buy + + self.edge_config = self.config.get('edge', {}) + self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + + self._total_capital: float = self.config['stake_amount'] + self._allowed_risk: float = self.edge_config.get('allowed_risk') + self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) + self._last_updated: int = 0 # Timestamp of pairs last updated time + + self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) + self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) + self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) + + # calculating stoploss range + self._stoploss_range = np.arange( + self._stoploss_range_min, + self._stoploss_range_max, + self._stoploss_range_step + ) + + self._timerange: TimeRange = Arguments.parse_timerange("%s-" % arrow.now().shift( + days=-1 * self._since_number_of_days).format('YYYYMMDD')) + + self.fee = self.exchange.get_fee() + + def calculate(self) -> bool: + pairs = self.config['exchange']['pair_whitelist'] + heartbeat = self.edge_config.get('process_throttle_secs') + + if (self._last_updated > 0) and ( + self._last_updated + heartbeat > arrow.utcnow().timestamp): + return False + + data: Dict[str, Any] = {} + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + logger.info('Using local backtesting data (using whitelist in given config) ...') + + data = optimize.load_data( + self.config['datadir'], + pairs=pairs, + ticker_interval=self.ticker_interval, + refresh_pairs=True, + exchange=self.exchange, + timerange=self._timerange + ) + + if not data: + # Reinitializing cached pairs + self._cached_pairs = {} + logger.critical("No data found. Edge is stopped ...") + return False + + preprocessed = self.tickerdata_to_dataframe(data) + + # Print timeframe + min_date, max_date = self.get_timeframe(preprocessed) + logger.info( + 'Measuring data from %s up to %s (%s days) ...', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] + + trades: list = [] + for pair, pair_data in preprocessed.items(): + # Sorting dataframe by date and reset index + pair_data = pair_data.sort_values(by=['date']) + pair_data = pair_data.reset_index(drop=True) + + ticker_data = self.advise_sell( + self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() + + trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range) + + # If no trade found then exit + if len(trades) == 0: + return False + + # Fill missing, calculable columns, profit, duration , abs etc. + trades_df = self._fill_calculable_fields(DataFrame(trades)) + self._cached_pairs = self._process_expectancy(trades_df) + self._last_updated = arrow.utcnow().timestamp + + # Not a nice hack but probably simplest solution: + # When backtest load data it loads the delta between disk and exchange + # The problem is that exchange consider that recent. + # it is but it is incomplete (c.f. _async_get_candle_history) + # So it causes get_signal to exit cause incomplete ticker_hist + # A patch to that would be update _pairs_last_refresh_time of exchange + # so it will download again all pairs + # Another solution is to add new data to klines instead of reassigning it: + # self.klines[pair].update(data) instead of self.klines[pair] = data in exchange package. + # But that means indexing timestamp and having a verification so that + # there is no empty range between two timestaps (recently added and last + # one) + self.exchange._pairs_last_refresh_time = {} + + return True + + def stake_amount(self, pair: str) -> float: + stoploss = self._cached_pairs[pair].stoploss + allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) + position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) + return position_size + + def stoploss(self, pair: str) -> float: + return self._cached_pairs[pair].stoploss + + def adjust(self, pairs) -> list: + """ + Filters out and sorts "pairs" according to Edge calculated pairs + """ + + final = [] + for pair, info in self._cached_pairs.items(): + if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ + info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ + pair in pairs: + final.append(pair) + + if final: + logger.info('Edge validated only %s', final) + else: + logger.info('Edge removed all pairs as no pair with minimum expectancy was found !') + + return final + + def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: + """ + The result frame contains a number of columns that are calculable + from other columns. These are left blank till all rows are added, + to be populated in single vector calls. + + Columns to be populated are: + - Profit + - trade duration + - profit abs + :param result Dataframe + :return: result Dataframe + """ + + # stake and fees + # stake = 0.015 + # 0.05% is 0.0005 + # fee = 0.001 + + stake = self.config.get('stake_amount') + fee = self.fee + + open_fee = fee / 2 + close_fee = fee / 2 + + result['trade_duration'] = result['close_time'] - result['open_time'] + + result['trade_duration'] = result['trade_duration'].map( + lambda x: int(x.total_seconds() / 60)) + + # Spends, Takes, Profit, Absolute Profit + + # Buy Price + result['buy_vol'] = stake / result['open_rate'] # How many target are we buying + result['buy_fee'] = stake * open_fee + result['buy_spend'] = stake + result['buy_fee'] # How much we're spending + + # Sell price + result['sell_sum'] = result['buy_vol'] * result['close_rate'] + result['sell_fee'] = result['sell_sum'] * close_fee + result['sell_take'] = result['sell_sum'] - result['sell_fee'] + + # profit_percent + result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend'] + + # Absolute profit + result['profit_abs'] = result['sell_take'] - result['buy_spend'] + + return result + + def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]: + """ + This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs + The calulation will be done per pair and per strategy. + """ + # Removing pairs having less than min_trades_number + min_trades_number = self.edge_config.get('min_trade_number', 10) + results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number) + ################################### + + # Removing outliers (Only Pumps) from the dataset + # The method to detect outliers is to calculate standard deviation + # Then every value more than (standard deviation + 2*average) is out (pump) + # + # Removing Pumps + if self.edge_config.get('remove_pumps', False): + results = results.groupby(['pair', 'stoploss']).apply( + lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()]) + ########################################################################## + + # Removing trades having a duration more than X minutes (set in config) + max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440) + results = results[results.trade_duration < max_trade_duration] + ####################################################################### + + if results.empty: + return {} + + groupby_aggregator = { + 'profit_abs': [ + ('nb_trades', 'count'), # number of all trades + ('profit_sum', lambda x: x[x > 0].sum()), # cumulative profit of all winning trades + ('loss_sum', lambda x: abs(x[x < 0].sum())), # cumulative loss of all losing trades + ('nb_win_trades', lambda x: x[x > 0].count()) # number of winning trades + ], + 'trade_duration': [('avg_trade_duration', 'mean')] + } + + # Group by (pair and stoploss) by applying above aggregator + df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( + groupby_aggregator).reset_index(col_level=1) + + # Dropping level 0 as we don't need it + df.columns = df.columns.droplevel(0) + + # Calculating number of losing trades, average win and average loss + df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades'] + df['average_win'] = df['profit_sum'] / df['nb_win_trades'] + df['average_loss'] = df['loss_sum'] / df['nb_loss_trades'] + + # Win rate = number of profitable trades / number of trades + df['winrate'] = df['nb_win_trades'] / df['nb_trades'] + + # risk_reward_ratio = average win / average loss + df['risk_reward_ratio'] = df['average_win'] / df['average_loss'] + + # required_risk_reward = (1 / winrate) - 1 + df['required_risk_reward'] = (1 / df['winrate']) - 1 + + # expectancy = (risk_reward_ratio * winrate) - (lossrate) + df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate']) + + # sort by expectancy and stoploss + df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( + 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() + + final = {} + for x in df.itertuples(): + info = { + 'stoploss': x.stoploss, + 'winrate': x.winrate, + 'risk_reward_ratio': x.risk_reward_ratio, + 'required_risk_reward': x.required_risk_reward, + 'expectancy': x.expectancy + } + final[x.pair] = self._pair_info(**info) + + # Returning a list of pairs in order of "expectancy" + return final + + def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): + buy_column = ticker_data['buy'].values + sell_column = ticker_data['sell'].values + date_column = ticker_data['date'].values + ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values + + result: list = [] + for stoploss in stoploss_range: + result += self._detect_next_stop_or_sell_point( + buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair + ) + + return result + + def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column, + ohlc_columns, stoploss, pair, start_point=0): + """ + Iterate through ohlc_columns recursively in order to find the next trade + Next trade opens from the first buy signal noticed to + The sell or stoploss signal after it. + It then calls itself cutting OHLC, buy_column, sell_colum and date_column + Cut from (the exit trade index) + 1 + Author: https://github.com/mishaker + """ + + result: list = [] + open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) + + # return empty if we don't find trade entry (i.e. buy==1) or + # we find a buy but at the of array + if open_trade_index == -1 or open_trade_index == len(buy_column) - 1: + return [] + else: + open_trade_index += 1 # when a buy signal is seen, + # trade opens in reality on the next candle + + stop_price_percentage = stoploss + 1 + open_price = ohlc_columns[open_trade_index, 0] + stop_price = (open_price * stop_price_percentage) + + # Searching for the index where stoploss is hit + stop_index = utf1st.find_1st( + ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller) + + # If we don't find it then we assume stop_index will be far in future (infinite number) + if stop_index == -1: + stop_index = float('inf') + + # Searching for the index where sell is hit + sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal) + + # If we don't find it then we assume sell_index will be far in future (infinite number) + if sell_index == -1: + sell_index = float('inf') + + # Check if we don't find any stop or sell point (in that case trade remains open) + # It is not interesting for Edge to consider it so we simply ignore the trade + # And stop iterating there is no more entry + if stop_index == sell_index == float('inf'): + return [] + + if stop_index <= sell_index: + exit_index = open_trade_index + stop_index + exit_type = SellType.STOP_LOSS + exit_price = stop_price + elif stop_index > sell_index: + # if exit is SELL then we exit at the next candle + exit_index = open_trade_index + sell_index + 1 + + # check if we have the next candle + if len(ohlc_columns) - 1 < exit_index: + return [] + + exit_type = SellType.SELL_SIGNAL + exit_price = ohlc_columns[exit_index, 0] + + trade = {'pair': pair, + 'stoploss': stoploss, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': date_column[open_trade_index], + 'close_time': date_column[exit_index], + 'open_index': start_point + open_trade_index, + 'close_index': start_point + exit_index, + 'trade_duration': '', + 'open_rate': round(open_price, 15), + 'close_rate': round(exit_price, 15), + 'exit_type': exit_type + } + + result.append(trade) + + # Calling again the same function recursively but giving + # it a view of exit_index till the end of array + return result + self._detect_next_stop_or_sell_point( + buy_column[exit_index:], + sell_column[exit_index:], + date_column[exit_index:], + ohlc_columns[exit_index:], + stoploss, + pair, + (start_point + exit_index) + ) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 51160332d..4a17e889e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -17,6 +17,7 @@ from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange +from freqtrade.edge import Edge from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State @@ -24,6 +25,7 @@ from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.exchange.exchange_helpers import order_book_to_dataframe + logger = logging.getLogger(__name__) @@ -54,6 +56,11 @@ class FreqtradeBot(object): self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = Exchange(self.config) + + # Initializing Edge only if enabled + self.edge = Edge(self.config, self.exchange, self.strategy) if \ + self.config.get('edge', {}).get('enabled', False) else None + self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] self._init_modules() @@ -179,6 +186,17 @@ class FreqtradeBot(object): # Keep only the subsets of pairs wanted (up to nb_assets) self.active_pair_whitelist = sanitized_list[:nb_assets] if nb_assets else sanitized_list + # Calculating Edge positiong + # Should be called before refresh_tickers + # Otherwise it will override cached klines in exchange + # with delta value (klines only from last refresh_pairs) + if self.edge: + self.edge.calculate() + self.active_pair_whitelist = self.edge.adjust(self.active_pair_whitelist) + + # Refreshing candles + self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) + # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -309,13 +327,17 @@ class FreqtradeBot(object): return used_rate - def _get_trade_stake_amount(self) -> Optional[float]: + def _get_trade_stake_amount(self, pair) -> Optional[float]: """ Check if stake amount can be fulfilled with the available balance for the stake currency :return: float: Stake Amount """ - stake_amount = self.config['stake_amount'] + if self.edge: + stake_amount = self.edge.stake_amount(pair) + else: + stake_amount = self.config['stake_amount'] + avaliable_amount = self.exchange.get_balance(self.config['stake_currency']) if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: @@ -373,15 +395,6 @@ class FreqtradeBot(object): :return: True if a trade object has been created and persisted, False otherwise """ interval = self.strategy.ticker_interval - stake_amount = self._get_trade_stake_amount() - - if not stake_amount: - return False - - logger.info( - 'Checking buy signals to create a new trade with stake_amount: %f ...', - stake_amount - ) whitelist = copy.deepcopy(self.active_pair_whitelist) # Remove currently opened and latest pairs from whitelist @@ -394,10 +407,18 @@ class FreqtradeBot(object): raise DependencyException('No currency pairs in whitelist') # running get_signal on historical data fetched - # to find buy signals for _pair in whitelist: (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) if buy and not sell: + stake_amount = self._get_trade_stake_amount(_pair) + if not stake_amount: + return False + + logger.info( + 'Buy signal found: about create a new trade with stake_amount: %f ...', + stake_amount + ) + bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ get('check_depth_of_market', {}) if (bidstrat_check_depth_of_market.get('enabled', False)) and\ @@ -624,10 +645,16 @@ class FreqtradeBot(object): return False def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: - should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + if self.edge: + stoploss = self.edge.stoploss(trade.pair) + should_sell = self.strategy.should_sell( + trade, sell_rate, datetime.utcnow(), buy, sell, force_stoploss=stoploss) + else: + should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + if should_sell.sell_flag: self.execute_sell(trade, sell_rate, should_sell.sell_type) - logger.info('excuted sell') + logger.info('executed sell, reason: %s', should_sell.sell_type) return True return False diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index c3cbce2e7..7e7b60ebc 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -410,7 +410,7 @@ class RPC(object): raise RPCException(f'position for {pair} already open - id: {trade.id}') # gen stake amount - stakeamount = self._freqtrade._get_trade_stake_amount() + stakeamount = self._freqtrade._get_trade_stake_amount(pair) # execute buy if self._freqtrade.execute_buy(pair, stakeamount, price): diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 27da6147c..212559c8c 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -203,17 +203,20 @@ class IStrategy(ABC): return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, - sell: bool, low: float = None, high: float = None) -> SellCheckTuple: + sell: bool, low: float = None, high: float = None, + force_stoploss: float = 0) -> SellCheckTuple: """ This function evaluate if on the condition required to trigger a sell has been reached if the threshold is reached and updates the trade record. :return: True if trade should be sold, False otherwise """ + # Set current rate to low for backtesting sell current_rate = low or rate current_profit = trade.calc_profit_percent(current_rate) stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, - current_time=date, current_profit=current_profit) + current_time=date, current_profit=current_profit, + force_stoploss=force_stoploss) if stoplossflag.sell_flag: return stoplossflag # Set current rate to low for backtesting sell @@ -241,7 +244,7 @@ class IStrategy(ABC): return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, - current_profit: float) -> SellCheckTuple: + current_profit: float, force_stoploss: float) -> SellCheckTuple: """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not @@ -250,7 +253,8 @@ class IStrategy(ABC): trailing_stop = self.config.get('trailing_stop', False) - trade.adjust_stop_loss(trade.open_rate, self.stoploss, initial=True) + trade.adjust_stop_loss(trade.open_rate, force_stoploss if force_stoploss + else self.stoploss, initial=True) # evaluate if the stoploss was hit if self.stoploss is not None and trade.stop_loss >= current_rate: diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index c6eeebbef..8a497725f 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -4,6 +4,7 @@ import logging from datetime import datetime from functools import reduce from typing import Dict, Optional +from collections import namedtuple from unittest.mock import MagicMock, PropertyMock import arrow @@ -12,6 +13,7 @@ from telegram import Chat, Message, Update from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange +from freqtrade.edge import Edge from freqtrade.freqtradebot import FreqtradeBot logging.getLogger('').setLevel(logging.INFO) @@ -42,7 +44,32 @@ def get_patched_exchange(mocker, config, api_mock=None) -> Exchange: return exchange +def patch_edge(mocker) -> None: + # "ETH/BTC", + # "LTC/BTC", + # "XRP/BTC", + # "NEO/BTC" + pair_info = namedtuple( + 'pair_info', + 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value={ + 'NEO/BTC': pair_info(-0.20, 0.66, 3.71, 0.50, 1.71), + 'LTC/BTC': pair_info(-0.21, 0.66, 3.71, 0.50, 1.71), + } + )) + mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) + mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value=True)) + + +def get_patched_edge(mocker, config) -> Edge: + patch_edge(mocker) + edge = Edge(config) + return edge + # Functions for recurrent object patching + + def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: """ This function patch _init_modules() to not call dependencies @@ -752,3 +779,23 @@ def buy_order_fee(): 'status': 'closed', 'fee': None } + + +@pytest.fixture(scope="function") +def edge_conf(default_conf): + default_conf['edge'] = { + "enabled": True, + "process_throttle_secs": 1800, + "calculate_since_number_of_days": 14, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "maximum_winrate": 0.80, + "minimum_expectancy": 0.20, + "min_trade_number": 15, + "max_trade_duration_minute": 1440, + "remove_pumps": False + } + + return default_conf diff --git a/freqtrade/tests/edge/__init__.py b/freqtrade/tests/edge/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py new file mode 100644 index 000000000..a7b1882a5 --- /dev/null +++ b/freqtrade/tests/edge/test_edge.py @@ -0,0 +1,310 @@ +# pragma pylint: disable=missing-docstring, C0103, C0330 +# pragma pylint: disable=protected-access, too-many-lines, invalid-name, too-many-arguments + +import pytest +import logging +from freqtrade.tests.conftest import get_patched_freqtradebot +from freqtrade.edge import Edge +from pandas import DataFrame, to_datetime +from freqtrade.strategy.interface import SellType +from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, + _get_frame_time_from_offset) +import arrow +import numpy as np +import math + +from unittest.mock import MagicMock + +# Cases to be tested: +# 1) Open trade should be removed from the end +# 2) Two complete trades within dataframe (with sell hit for all) +# 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss +# 4) Entered, sl 3%, candle drops 4%, recovers to 1% => Trade closed, 3% loss +# 5) Stoploss and sell are hit. should sell on stoploss +#################################################################### + +ticker_start_time = arrow.get(2018, 10, 3) +ticker_interval_in_minute = 60 +_ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} + + +# Open trade should be removed from the end +tc0 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 1]], # enter trade (signal on last candle) + stop_loss=-0.99, roi=float('inf'), profit_perc=0.00, + trades=[] +) + +# Two complete trades within dataframe(with sell hit for all) +tc1 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 1], # enter trade (signal on last candle) + [2, 5000, 5025, 4975, 4987, 6172, 0, 0], # exit at open + [3, 5000, 5025, 4975, 4987, 6172, 1, 0], # no action + [4, 5000, 5025, 4975, 4987, 6172, 0, 0], # should enter the trade + [5, 5000, 5025, 4975, 4987, 6172, 0, 1], # no action + [6, 5000, 5025, 4975, 4987, 6172, 0, 0], # should sell +], + stop_loss=-0.99, roi=float('inf'), profit_perc=0.00, + trades=[BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=1, close_tick=2), + BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=4, close_tick=6)] +) + +# 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss +tc2 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4600, 4987, 6172, 0, 0], # enter trade, stoploss hit + [2, 5000, 5025, 4975, 4987, 6172, 0, 0], +], + stop_loss=-0.01, roi=float('inf'), profit_perc=-0.01, + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] +) + +# 4) Entered, sl 3 %, candle drops 4%, recovers to 1 % = > Trade closed, 3 % loss +tc3 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4800, 4987, 6172, 0, 0], # enter trade, stoploss hit + [2, 5000, 5025, 4975, 4987, 6172, 0, 0], +], + stop_loss=-0.03, roi=float('inf'), profit_perc=-0.03, + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] +) + +# 5) Stoploss and sell are hit. should sell on stoploss +tc4 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4800, 4987, 6172, 0, 1], # enter trade, stoploss hit, sell signal + [2, 5000, 5025, 4975, 4987, 6172, 0, 0], +], + stop_loss=-0.03, roi=float('inf'), profit_perc=-0.03, + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] +) + +TESTS = [ + tc0, + tc1, + tc2, + tc3, + tc4 +] + + +@pytest.mark.parametrize("data", TESTS) +def test_edge_results(edge_conf, mocker, caplog, data) -> None: + """ + run functional tests + """ + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + frame = _build_backtest_dataframe(data.data) + caplog.set_level(logging.DEBUG) + edge.fee = 0 + + trades = edge._find_trades_for_stoploss_range(frame, 'TEST/BTC', [data.stop_loss]) + results = edge._fill_calculable_fields(DataFrame(trades)) if trades else DataFrame() + + print(results) + + assert len(trades) == len(data.trades) + + if not results.empty: + assert round(results["profit_percent"].sum(), 3) == round(data.profit_perc, 3) + + for c, trade in enumerate(data.trades): + res = results.iloc[c] + assert res.exit_type == trade.sell_reason + assert res.open_time == _get_frame_time_from_offset(trade.open_tick) + assert res.close_time == _get_frame_time_from_offset(trade.close_tick) + + +def test_adjust(mocker, default_conf): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value={ + 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) + } + )) + + pairs = ['A/B', 'C/D', 'E/F', 'G/H'] + assert(edge.adjust(pairs) == ['E/F', 'C/D']) + + +def test_stoploss(mocker, default_conf): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value={ + 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) + } + )) + + assert edge.stoploss('E/F') == -0.01 + + +def _validate_ohlc(buy_ohlc_sell_matrice): + for index, ohlc in enumerate(buy_ohlc_sell_matrice): + # if not high < open < low or not high < close < low + if not ohlc[3] >= ohlc[2] >= ohlc[4] or not ohlc[3] >= ohlc[5] >= ohlc[4]: + raise Exception('Line ' + str(index + 1) + ' of ohlc has invalid values!') + return True + + +def _build_dataframe(buy_ohlc_sell_matrice): + _validate_ohlc(buy_ohlc_sell_matrice) + tickers = [] + for ohlc in buy_ohlc_sell_matrice: + ticker = { + 'date': ticker_start_time.shift( + minutes=( + ohlc[0] * + ticker_interval_in_minute)).timestamp * + 1000, + 'buy': ohlc[1], + 'open': ohlc[2], + 'high': ohlc[3], + 'low': ohlc[4], + 'close': ohlc[5], + 'sell': ohlc[6]} + tickers.append(ticker) + + frame = DataFrame(tickers) + frame['date'] = to_datetime(frame['date'], + unit='ms', + utc=True, + infer_datetime_format=True) + + return frame + + +def _time_on_candle(number): + return np.datetime64(ticker_start_time.shift( + minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') + + +def test_edge_heartbeat_calculate(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + heartbeat = edge_conf['edge']['process_throttle_secs'] + + # should not recalculate if heartbeat not reached + edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 + + assert edge.calculate() is False + + +def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, + timerange=None, exchange=None): + hz = 0.1 + base = 0.001 + + ETHBTC = [ + [ + ticker_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, + math.sin(x * hz) / 1000 + base, + math.sin(x * hz) / 1000 + base + 0.0001, + math.sin(x * hz) / 1000 + base - 0.0001, + math.sin(x * hz) / 1000 + base, + 123.45 + ] for x in range(0, 500)] + + hz = 0.2 + base = 0.002 + LTCBTC = [ + [ + ticker_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, + math.sin(x * hz) / 1000 + base, + math.sin(x * hz) / 1000 + base + 0.0001, + math.sin(x * hz) / 1000 + base - 0.0001, + math.sin(x * hz) / 1000 + base, + 123.45 + ] for x in range(0, 500)] + + pairdata = {'NEO/BTC': ETHBTC, 'LTC/BTC': LTCBTC} + return pairdata + + +def test_edge_process_downloaded_data(mocker, default_conf): + default_conf['datadir'] = None + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.optimize.load_data', mocked_load_data) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) + + assert edge.calculate() + assert len(edge._cached_pairs) == 2 + assert edge._last_updated <= arrow.utcnow().timestamp + 2 + + +def test_process_expectancy(mocker, edge_conf): + edge_conf['edge']['min_trade_number'] = 2 + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + + def get_fee(): + return 0.001 + + freqtrade.exchange.get_fee = get_fee + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + + trades = [ + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': np.datetime64('2018-10-03T00:05:00.000000000'), + 'close_time': np.datetime64('2018-10-03T00:10:00.000000000'), + 'open_index': 1, + 'close_index': 1, + 'trade_duration': '', + 'open_rate': 17, + 'close_rate': 17, + 'exit_type': 'sell_signal'}, + + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': np.datetime64('2018-10-03T00:20:00.000000000'), + 'close_time': np.datetime64('2018-10-03T00:25:00.000000000'), + 'open_index': 4, + 'close_index': 4, + 'trade_duration': '', + 'open_rate': 20, + 'close_rate': 20, + 'exit_type': 'sell_signal'}, + + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': np.datetime64('2018-10-03T00:30:00.000000000'), + 'close_time': np.datetime64('2018-10-03T00:40:00.000000000'), + 'open_index': 6, + 'close_index': 7, + 'trade_duration': '', + 'open_rate': 26, + 'close_rate': 34, + 'exit_type': 'sell_signal'} + ] + + trades_df = DataFrame(trades) + trades_df = edge._fill_calculable_fields(trades_df) + final = edge._process_expectancy(trades_df) + assert len(final) == 1 + + assert 'TEST/BTC' in final + assert final['TEST/BTC'].stoploss == -0.9 + assert round(final['TEST/BTC'].winrate, 10) == 0.3333333333 + assert round(final['TEST/BTC'].risk_reward_ratio, 10) == 306.5384615384 + assert round(final['TEST/BTC'].required_risk_reward, 10) == 2.0 + assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128 diff --git a/freqtrade/tests/optimize/__init__.py b/freqtrade/tests/optimize/__init__.py index 2b7222e88..3d3066950 100644 --- a/freqtrade/tests/optimize/__init__.py +++ b/freqtrade/tests/optimize/__init__.py @@ -31,7 +31,7 @@ class BTContainer(NamedTuple): def _get_frame_time_from_offset(offset): return ticker_start_time.shift( - minutes=(offset * ticker_interval_in_minute)).datetime + minutes=(offset * ticker_interval_in_minute)).datetime.replace(tzinfo=None) def _build_backtest_dataframe(ticker_with_signals): diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 806c136bc..eaec3bf49 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -1,4 +1,4 @@ -# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, C0330, unused-argument import logging from unittest.mock import MagicMock @@ -34,15 +34,15 @@ tc0 = BTContainer(data=[ # TC2: Stop-Loss Triggered 3% Loss tc1 = BTContainer(data=[ # D O H L C V B S - [0, 5000, 5025, 4975, 4987, 6172, 1, 0], - [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) - [2, 4987, 5012, 4962, 4975, 6172, 0, 0], - [3, 4975, 5000, 4800, 4962, 6172, 0, 0], # exit with stoploss hit - [4, 4962, 4987, 4937, 4950, 6172, 0, 0], - [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4962, 4975, 6172, 0, 0], + [3, 4975, 5000, 4800, 4962, 6172, 0, 0], # exit with stoploss hit + [4, 4962, 4987, 4937, 4950, 6172, 0, 0], + [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], stop_loss=-0.03, roi=1, profit_perc=-0.03, trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=3)] - ) +) # Test 3 Candle drops 4%, Recovers 1%. @@ -127,7 +127,7 @@ tc6 = BTContainer(data=[ [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], stop_loss=-0.02, roi=0.03, profit_perc=0.03, trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=2)] - ) +) TESTS = [ tc0, diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 55479cc6f..4cba9e308 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -18,7 +18,7 @@ from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType, SellCheckTuple -from freqtrade.tests.conftest import log_has, patch_exchange +from freqtrade.tests.conftest import log_has, patch_exchange, patch_edge # Functions for recurrent object patching @@ -177,7 +177,7 @@ def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mock freqtrade = FreqtradeBot(default_conf) - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('ETH/BTC') assert result == default_conf['stake_amount'] @@ -195,7 +195,7 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, freqtrade = FreqtradeBot(default_conf) with pytest.raises(DependencyException, match=r'.*stake amount.*'): - freqtrade._get_trade_stake_amount() + freqtrade._get_trade_stake_amount('ETH/BTC') def test_get_trade_stake_amount_unlimited_amount(default_conf, @@ -224,28 +224,131 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, patch_get_signal(freqtrade) # no open trades, order amount should be 'balance / max_open_trades' - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('ETH/BTC') assert result == default_conf['stake_amount'] / conf['max_open_trades'] # create one trade, order amount should be 'balance / (max_open_trades - num_open_trades)' freqtrade.create_trade() - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('LTC/BTC') assert result == default_conf['stake_amount'] / (conf['max_open_trades'] - 1) # create 2 trades, order amount should be None freqtrade.create_trade() - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('XRP/BTC') assert result is None # set max_open_trades = None, so do not trade conf['max_open_trades'] = 0 freqtrade = FreqtradeBot(conf) - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('NEO/BTC') assert result is None +def test_edge_called_in_process(mocker, edge_conf) -> None: + patch_RPCManager(mocker) + patch_edge(mocker) + + def _refresh_whitelist(list): + return ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'] + + patch_exchange(mocker) + freqtrade = FreqtradeBot(edge_conf) + freqtrade._refresh_whitelist = _refresh_whitelist + patch_get_signal(freqtrade) + freqtrade._process() + assert freqtrade.active_pair_whitelist == ['NEO/BTC', 'LTC/BTC'] + + +def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + patch_edge(mocker) + freqtrade = FreqtradeBot(edge_conf) + + assert freqtrade._get_trade_stake_amount('NEO/BTC') == (0.001 * 0.01) / 0.20 + assert freqtrade._get_trade_stake_amount('LTC/BTC') == (0.001 * 0.01) / 0.20 + + +def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, edge_conf) -> None: + + patch_RPCManager(mocker) + patch_exchange(mocker) + patch_edge(mocker) + + # Strategy stoploss is -0.1 but Edge imposes a stoploss at -0.2 + # Thus, if price falls 21%, stoploss should be triggered + # + # mocking the ticker: price is falling ... + buy_price = limit_buy_order['price'] + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=MagicMock(return_value={ + 'bid': buy_price * 0.79, + 'ask': buy_price * 0.79, + 'last': buy_price * 0.79 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets, + ) + ############################################# + + # Create a trade with "limit_buy_order" price + freqtrade = FreqtradeBot(edge_conf) + freqtrade.active_pair_whitelist = ['NEO/BTC'] + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.create_trade() + trade = Trade.query.first() + trade.update(limit_buy_order) + ############################################# + + # stoploss shoud be hit + assert freqtrade.handle_trade(trade) is True + assert log_has('executed sell, reason: SellType.STOP_LOSS', caplog.record_tuples) + assert trade.sell_reason == SellType.STOP_LOSS.value + + +def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, + mocker, edge_conf) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + patch_edge(mocker) + + # Strategy stoploss is -0.1 but Edge imposes a stoploss at -0.2 + # Thus, if price falls 15%, stoploss should not be triggered + # + # mocking the ticker: price is falling ... + buy_price = limit_buy_order['price'] + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=MagicMock(return_value={ + 'bid': buy_price * 0.85, + 'ask': buy_price * 0.85, + 'last': buy_price * 0.85 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets, + ) + ############################################# + + # Create a trade with "limit_buy_order" price + freqtrade = FreqtradeBot(edge_conf) + freqtrade.active_pair_whitelist = ['NEO/BTC'] + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.create_trade() + trade = Trade.query.first() + trade.update(limit_buy_order) + ############################################# + + # stoploss shoud not be hit + assert freqtrade.handle_trade(trade) is False + + def test_get_min_pair_stake_amount(mocker, default_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -494,7 +597,7 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, patch_get_signal(freqtrade) assert freqtrade.create_trade() is False - assert freqtrade._get_trade_stake_amount() is None + assert freqtrade._get_trade_stake_amount('ETH/BTC') is None def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: @@ -593,7 +696,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, assert trade.amount == 90.99181073703367 assert log_has( - 'Checking buy signals to create a new trade with stake_amount: 0.001000 ...', + 'Buy signal found: about create a new trade with stake_amount: 0.001000 ...', caplog.record_tuples ) @@ -1547,7 +1650,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.stop_loss_reached = \ - lambda current_rate, trade, current_time, current_profit: SellCheckTuple( + lambda current_rate, trade, current_time, force_stoploss, current_profit: SellCheckTuple( sell_flag=False, sell_type=SellType.NONE) freqtrade.create_trade() @@ -1821,7 +1924,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca exchange='binance', open_rate=0.245441, open_order_id="123456" - ) + ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2097,9 +2200,9 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) """ patch_exchange(mocker) mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - get_markets=markets, - get_order_book=order_book_l2 + 'freqtrade.exchange.Exchange', + get_markets=markets, + get_order_book=order_book_l2 ) default_conf['exchange']['name'] = 'binance' default_conf['bid_strategy']['use_order_book'] = True diff --git a/requirements.txt b/requirements.txt index 2c0011aea..374d6fce0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,9 @@ scikit-optimize==0.5.2 # Required for plotting data #plotly==3.1.1 + +# find first, C search in arrays +py_find_1st==1.1.2 + +#Load ticker files 30% faster +ujson==1.35 diff --git a/setup.py b/setup.py index 8853ef7f8..c5f61c34d 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,8 @@ setup(name='freqtrade', 'cachetools', 'coinmarketcap', 'scikit-optimize', + 'ujson', + 'py_find_1st' ], include_package_data=True, zip_safe=False,