Merge branch 'develop' into feat_readjust_entry

This commit is contained in:
eSeR1805 2022-05-01 21:42:15 +03:00
commit 04c51d2d1a
No known key found for this signature in database
GPG Key ID: BA53686259B46936
53 changed files with 689 additions and 388 deletions

View File

@ -1,9 +1,9 @@
Thank you for sending your pull request. But first, have you included
<!-- Thank you for sending your pull request. But first, have you included
unit tests, and is your code PEP8 conformant? [More details](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md)
-->
## Summary
Explain in one sentence the goal of this PR
<!-- Explain in one sentence the goal of this PR -->
Solve the issue: #___
@ -14,4 +14,4 @@ Solve the issue: #___
## What's new?
*Explain in details what this PR solve or improve. You can include visuals.*
<!-- Explain in details what this PR solve or improve. You can include visuals. -->

View File

@ -30,6 +30,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
[--data-format-ohlcv {json,jsongz,hdf5}]
[--data-format-trades {json,jsongz,hdf5}]
[--trading-mode {spot,margin,futures}]
[--prepend]
optional arguments:
-h, --help show this help message and exit
@ -62,6 +63,7 @@ optional arguments:
`jsongz`).
--trading-mode {spot,margin,futures}
Select Trading mode
--prepend Allow data prepending.
Common arguments:
-v, --verbose Verbose mode (-vv for more, -vvv to get all messages).
@ -157,10 +159,21 @@ freqtrade download-data --exchange binance --pairs .*/USDT
- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.)
- To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`.
- To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days).
- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. Eventually set end dates are ignored.
- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020.
- Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data.
- To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options.
#### Download additional data before the current timerange
Assuming you downloaded all data from 2022 (`--timerange 20220101-`) - but you'd now like to also backtest with earlier data.
You can do so by using the `--prepend` flag, combined with `--timerange` - specifying an end-date.
``` bash
freqtrade download-data --exchange binance --pairs ETH/USDT XRP/USDT BTC/USDT --prepend --timerange 20210101-20220101
```
!!! Note
Freqtrade will ignore the end-date in this mode if data is available, updating the end-date to the existing data start point.
### Data format

View File

@ -200,11 +200,12 @@ For that reason, they must implement the following methods:
* `global_stop()`
* `stop_per_pair()`.
`global_stop()` and `stop_per_pair()` must return a ProtectionReturn tuple, which consists of:
`global_stop()` and `stop_per_pair()` must return a ProtectionReturn object, which consists of:
* lock pair - boolean
* lock until - datetime - until when should the pair be locked (will be rounded up to the next new candle)
* reason - string, used for logging and storage in the database
* lock_side - long, short or '*'.
The `until` portion should be calculated using the provided `calculate_lock_end()` method.

View File

@ -48,6 +48,8 @@ If `trade_limit` or more trades resulted in stoploss, trading will stop for `sto
This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time.
Similarly, this protection will by default look at all trades (long and short). For futures bots, setting `only_per_side` will make the bot only consider one side, and will then only lock this one side, allowing for example shorts to continue after a series of long stoplosses.
The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles.
``` python
@ -59,7 +61,8 @@ def protections(self):
"lookback_period_candles": 24,
"trade_limit": 4,
"stop_duration_candles": 4,
"only_per_pair": False
"only_per_pair": False,
"only_per_side": False
}
]
```

View File

@ -72,7 +72,8 @@ ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive",
"timerange", "download_trades", "exchange", "timeframes",
"erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode"]
"erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode",
"prepend_data"]
ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
"db_url", "trade_source", "export", "exportfilename",

View File

@ -443,6 +443,11 @@ AVAILABLE_CLI_OPTIONS = {
default=['1m', '5m'],
nargs='+',
),
"prepend_data": Arg(
'--prepend',
help='Allow data prepending.',
action='store_true',
),
"erase": Arg(
'--erase',
help='Clean all existing data for the selected exchange/pairs/timeframes.',

View File

@ -85,6 +85,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
new_pairs_days=config['new_pairs_days'],
erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'],
trading_mode=config.get('trading_mode', 'spot'),
prepend=config.get('prepend_data', False)
)
except KeyboardInterrupt:

View File

@ -22,6 +22,6 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str
# Ensure these modes are using Dry-run
config['dry_run'] = True
validate_config_consistency(config)
validate_config_consistency(config, preliminary=True)
return config

View File

@ -39,7 +39,7 @@ def _extend_validator(validator_class):
FreqtradeValidator = _extend_validator(Draft4Validator)
def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]:
def validate_config_schema(conf: Dict[str, Any], preliminary: bool = False) -> Dict[str, Any]:
"""
Validate the configuration follow the Config Schema
:param conf: Config in JSON format
@ -49,7 +49,10 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]:
if conf.get('runmode', RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE):
conf_schema['required'] = constants.SCHEMA_TRADE_REQUIRED
elif conf.get('runmode', RunMode.OTHER) in (RunMode.BACKTEST, RunMode.HYPEROPT):
conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED
if preliminary:
conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED
else:
conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED_FINAL
else:
conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED
try:
@ -64,7 +67,7 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]:
)
def validate_config_consistency(conf: Dict[str, Any]) -> None:
def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False) -> None:
"""
Validate the configuration consistency.
Should be ran after loading both configuration and strategy,
@ -85,7 +88,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None:
# validate configuration before returning
logger.info('Validating configuration ...')
validate_config_schema(conf)
validate_config_schema(conf, preliminary=preliminary)
def _validate_unlimited_amount(conf: Dict[str, Any]) -> None:

View File

@ -393,6 +393,8 @@ class Configuration:
self._args_to_config(config, argname='trade_source',
logstring='Using trades from: {}')
self._args_to_config(config, argname='prepend_data',
logstring='Prepend detected. Allowing data prepending.')
self._args_to_config(config, argname='erase',
logstring='Erase detected. Deleting existing data.')

View File

@ -462,6 +462,10 @@ SCHEMA_BACKTEST_REQUIRED = [
'dataformat_ohlcv',
'dataformat_trades',
]
SCHEMA_BACKTEST_REQUIRED_FINAL = SCHEMA_BACKTEST_REQUIRED + [
'stoploss',
'minimal_roi',
]
SCHEMA_MINIMAL_REQUIRED = [
'exchange',

View File

@ -5,7 +5,7 @@ import logging
from copy import copy
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union
from typing import Any, Dict, List, Optional, Union
import numpy as np
import pandas as pd
@ -400,168 +400,3 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame,
trades = trades.loc[(trades['open_date'] >= trades_start) &
(trades['close_date'] <= trades_stop)]
return trades
def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float:
"""
Calculate market change based on "column".
Calculation is done by taking the first non-null and the last non-null element of each column
and calculating the pctchange as "(last - first) / first".
Then the results per pair are combined as mean.
:param data: Dict of Dataframes, dict key should be pair.
:param column: Column in the original dataframes to use
:return:
"""
tmp_means = []
for pair, df in data.items():
start = df[column].dropna().iloc[0]
end = df[column].dropna().iloc[-1]
tmp_means.append((end - start) / start)
return float(np.mean(tmp_means))
def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame],
column: str = "close") -> pd.DataFrame:
"""
Combine multiple dataframes "column"
:param data: Dict of Dataframes, dict key should be pair.
:param column: Column in the original dataframes to use
:return: DataFrame with the column renamed to the dict key, and a column
named mean, containing the mean of all pairs.
:raise: ValueError if no data is provided.
"""
df_comb = pd.concat([data[pair].set_index('date').rename(
{column: pair}, axis=1)[pair] for pair in data], axis=1)
df_comb['mean'] = df_comb.mean(axis=1)
return df_comb
def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
timeframe: str) -> pd.DataFrame:
"""
Adds a column `col_name` with the cumulative profit for the given trades array.
:param df: DataFrame with date index
:param trades: DataFrame containing trades (requires columns close_date and profit_abs)
:param col_name: Column name that will be assigned the results
:param timeframe: Timeframe used during the operations
:return: Returns df with one additional column, col_name, containing the cumulative profit.
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
from freqtrade.exchange import timeframe_to_minutes
timeframe_minutes = timeframe_to_minutes(timeframe)
# Resample to timeframe to make sure trades match candles
_trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
)[['profit_abs']].sum()
df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum()
# Set first value to 0
df.loc[df.iloc[0].name, col_name] = 0
# FFill to get continuous
df[col_name] = df[col_name].ffill()
return df
def _calc_drawdown_series(profit_results: pd.DataFrame, *, date_col: str, value_col: str
) -> pd.DataFrame:
max_drawdown_df = pd.DataFrame()
max_drawdown_df['cumulative'] = profit_results[value_col].cumsum()
max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax()
max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value']
max_drawdown_df['date'] = profit_results.loc[:, date_col]
return max_drawdown_df
def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_ratio'
):
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
:param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio')
:return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown,
high and low time and high and low value.
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
profit_results = trades.sort_values(date_col).reset_index(drop=True)
max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col)
return max_drawdown_df
def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_abs', starting_balance: float = 0
) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float, float]:
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
:param value_col: Column in DataFrame to use for values (defaults to 'profit_abs')
:param starting_balance: Portfolio starting balance - properly calculate relative drawdown.
:return: Tuple (float, highdate, lowdate, highvalue, lowvalue, relative_drawdown)
with absolute max drawdown, high and low time and high and low value,
and the relative account drawdown
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
profit_results = trades.sort_values(date_col).reset_index(drop=True)
max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col)
idxmin = max_drawdown_df['drawdown'].idxmin()
if idxmin == 0:
raise ValueError("No losing trade, therefore no drawdown.")
high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col]
low_date = profit_results.loc[idxmin, date_col]
high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin]
['high_value'].idxmax(), 'cumulative']
low_val = max_drawdown_df.loc[idxmin, 'cumulative']
max_drawdown_rel = 0.0
if high_val + starting_balance != 0:
max_drawdown_rel = (high_val - low_val) / (high_val + starting_balance)
return (
abs(min(max_drawdown_df['drawdown'])),
high_date,
low_date,
high_val,
low_val,
max_drawdown_rel
)
def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]:
"""
Calculate min/max cumsum of trades, to show if the wallet/stake amount ratio is sane
:param trades: DataFrame containing trades (requires columns close_date and profit_percent)
:param starting_balance: Add starting balance to results, to show the wallets high / low points
:return: Tuple (float, float) with cumsum of profit_abs
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
csum_df = pd.DataFrame()
csum_df['sum'] = trades['profit_abs'].cumsum()
csum_min = csum_df['sum'].min() + starting_balance
csum_max = csum_df['sum'].max() + starting_balance
return csum_min, csum_max
def calculate_cagr(days_passed: int, starting_balance: float, final_balance: float) -> float:
"""
Calculate CAGR
:param days_passed: Days passed between start and ending balance
:param starting_balance: Starting balance
:param final_balance: Final balance to calculate CAGR against
:return: CAGR
"""
return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1

View File

@ -139,8 +139,9 @@ def _load_cached_data_for_updating(
timeframe: str,
timerange: Optional[TimeRange],
data_handler: IDataHandler,
candle_type: CandleType
) -> Tuple[DataFrame, Optional[int]]:
candle_type: CandleType,
prepend: bool = False,
) -> Tuple[DataFrame, Optional[int], Optional[int]]:
"""
Load cached data to download more data.
If timerange is passed in, checks whether data from an before the stored data will be
@ -150,9 +151,12 @@ def _load_cached_data_for_updating(
Note: Only used by download_pair_history().
"""
start = None
end = None
if timerange:
if timerange.starttype == 'date':
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
if timerange.stoptype == 'date':
end = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
# Intentionally don't pass timerange in - since we need to load the full dataset.
data = data_handler.ohlcv_load(pair, timeframe=timeframe,
@ -160,14 +164,17 @@ def _load_cached_data_for_updating(
drop_incomplete=True, warn_no_data=False,
candle_type=candle_type)
if not data.empty:
if start and start < data.iloc[0]['date']:
if not prepend and start and start < data.iloc[0]['date']:
# Earlier data than existing data requested, redownload all
data = DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS)
else:
start = data.iloc[-1]['date']
if prepend:
end = data.iloc[0]['date']
else:
start = data.iloc[-1]['date']
start_ms = int(start.timestamp() * 1000) if start else None
return data, start_ms
end_ms = int(end.timestamp() * 1000) if end else None
return data, start_ms, end_ms
def _download_pair_history(pair: str, *,
@ -180,6 +187,7 @@ def _download_pair_history(pair: str, *,
timerange: Optional[TimeRange] = None,
candle_type: CandleType,
erase: bool = False,
prepend: bool = False,
) -> bool:
"""
Download latest candles from the exchange for the pair and timeframe passed in parameters
@ -187,8 +195,6 @@ def _download_pair_history(pair: str, *,
exists in a cache. If timerange starts earlier than the data in the cache,
the full data will be redownloaded
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
:param pair: pair to download
:param timeframe: Timeframe (e.g "5m")
:param timerange: range of time to download
@ -203,14 +209,17 @@ def _download_pair_history(pair: str, *,
if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type):
logger.info(f'Deleting existing data for pair {pair}, {timeframe}, {candle_type}.')
logger.info(
f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe}, '
f'candle type: {candle_type} and store in {datadir}.'
)
data, since_ms, until_ms = _load_cached_data_for_updating(
pair, timeframe, timerange,
data_handler=data_handler,
candle_type=candle_type,
prepend=prepend)
data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange,
data_handler=data_handler,
candle_type=candle_type)
logger.info(f'({process}) - Download history data for "{pair}", {timeframe}, '
f'{candle_type} and store in {datadir}.'
f'From {format_ms_time(since_ms) if since_ms else "start"} to '
f'{format_ms_time(until_ms) if until_ms else "now"}'
)
logger.debug("Current Start: %s",
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
@ -225,6 +234,7 @@ def _download_pair_history(pair: str, *,
days=-new_pairs_days).int_timestamp * 1000,
is_new_pair=data.empty,
candle_type=candle_type,
until_ms=until_ms if until_ms else None
)
# TODO: Maybe move parsing to exchange class (?)
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
@ -257,6 +267,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
timerange: Optional[TimeRange] = None,
new_pairs_days: int = 30, erase: bool = False,
data_format: str = None,
prepend: bool = False,
) -> List[str]:
"""
Refresh stored ohlcv data for backtesting and hyperopt operations.
@ -280,7 +291,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
timerange=timerange, data_handler=data_handler,
timeframe=str(timeframe), new_pairs_days=new_pairs_days,
candle_type=candle_type,
erase=erase)
erase=erase, prepend=prepend)
if trading_mode == 'futures':
# Predefined candletype (and timeframe) depending on exchange
# Downloads what is necessary to backtest based on futures data.
@ -294,7 +305,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
timerange=timerange, data_handler=data_handler,
timeframe=str(tf_mark), new_pairs_days=new_pairs_days,
candle_type=funding_candle_type,
erase=erase)
erase=erase, prepend=prepend)
return pairs_not_available
@ -312,8 +323,9 @@ def _download_trades_history(exchange: Exchange,
try:
until = None
if (timerange and timerange.starttype == 'date'):
since = timerange.startts * 1000
if timerange:
if timerange.starttype == 'date':
since = timerange.startts * 1000
if timerange.stoptype == 'date':
until = timerange.stopts * 1000
else:

173
freqtrade/data/metrics.py Normal file
View File

@ -0,0 +1,173 @@
import logging
from typing import Dict, Tuple
import numpy as np
import pandas as pd
logger = logging.getLogger(__name__)
def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float:
"""
Calculate market change based on "column".
Calculation is done by taking the first non-null and the last non-null element of each column
and calculating the pctchange as "(last - first) / first".
Then the results per pair are combined as mean.
:param data: Dict of Dataframes, dict key should be pair.
:param column: Column in the original dataframes to use
:return:
"""
tmp_means = []
for pair, df in data.items():
start = df[column].dropna().iloc[0]
end = df[column].dropna().iloc[-1]
tmp_means.append((end - start) / start)
return float(np.mean(tmp_means))
def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame],
column: str = "close") -> pd.DataFrame:
"""
Combine multiple dataframes "column"
:param data: Dict of Dataframes, dict key should be pair.
:param column: Column in the original dataframes to use
:return: DataFrame with the column renamed to the dict key, and a column
named mean, containing the mean of all pairs.
:raise: ValueError if no data is provided.
"""
df_comb = pd.concat([data[pair].set_index('date').rename(
{column: pair}, axis=1)[pair] for pair in data], axis=1)
df_comb['mean'] = df_comb.mean(axis=1)
return df_comb
def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
timeframe: str) -> pd.DataFrame:
"""
Adds a column `col_name` with the cumulative profit for the given trades array.
:param df: DataFrame with date index
:param trades: DataFrame containing trades (requires columns close_date and profit_abs)
:param col_name: Column name that will be assigned the results
:param timeframe: Timeframe used during the operations
:return: Returns df with one additional column, col_name, containing the cumulative profit.
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
from freqtrade.exchange import timeframe_to_minutes
timeframe_minutes = timeframe_to_minutes(timeframe)
# Resample to timeframe to make sure trades match candles
_trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
)[['profit_abs']].sum()
df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum()
# Set first value to 0
df.loc[df.iloc[0].name, col_name] = 0
# FFill to get continuous
df[col_name] = df[col_name].ffill()
return df
def _calc_drawdown_series(profit_results: pd.DataFrame, *, date_col: str, value_col: str
) -> pd.DataFrame:
max_drawdown_df = pd.DataFrame()
max_drawdown_df['cumulative'] = profit_results[value_col].cumsum()
max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax()
max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value']
max_drawdown_df['date'] = profit_results.loc[:, date_col]
return max_drawdown_df
def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_ratio'
):
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
:param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio')
:return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown,
high and low time and high and low value.
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
profit_results = trades.sort_values(date_col).reset_index(drop=True)
max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col)
return max_drawdown_df
def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_abs', starting_balance: float = 0
) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float, float]:
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
:param value_col: Column in DataFrame to use for values (defaults to 'profit_abs')
:param starting_balance: Portfolio starting balance - properly calculate relative drawdown.
:return: Tuple (float, highdate, lowdate, highvalue, lowvalue, relative_drawdown)
with absolute max drawdown, high and low time and high and low value,
and the relative account drawdown
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
profit_results = trades.sort_values(date_col).reset_index(drop=True)
max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col)
idxmin = max_drawdown_df['drawdown'].idxmin()
if idxmin == 0:
raise ValueError("No losing trade, therefore no drawdown.")
high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col]
low_date = profit_results.loc[idxmin, date_col]
high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin]
['high_value'].idxmax(), 'cumulative']
low_val = max_drawdown_df.loc[idxmin, 'cumulative']
max_drawdown_rel = 0.0
if high_val + starting_balance != 0:
max_drawdown_rel = (high_val - low_val) / (high_val + starting_balance)
return (
abs(min(max_drawdown_df['drawdown'])),
high_date,
low_date,
high_val,
low_val,
max_drawdown_rel
)
def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]:
"""
Calculate min/max cumsum of trades, to show if the wallet/stake amount ratio is sane
:param trades: DataFrame containing trades (requires columns close_date and profit_percent)
:param starting_balance: Add starting balance to results, to show the wallets high / low points
:return: Tuple (float, float) with cumsum of profit_abs
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
csum_df = pd.DataFrame()
csum_df['sum'] = trades['profit_abs'].cumsum()
csum_min = csum_df['sum'].min() + starting_balance
csum_max = csum_df['sum'].max() + starting_balance
return csum_min, csum_max
def calculate_cagr(days_passed: int, starting_balance: float, final_balance: float) -> float:
"""
Calculate CAGR
:param days_passed: Days passed between start and ending balance
:param starting_balance: Starting balance
:param final_balance: Final balance to calculate CAGR against
:return: CAGR
"""
return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1

View File

@ -95,6 +95,7 @@ class Binance(Exchange):
async def _async_get_historic_ohlcv(self, pair: str, timeframe: str,
since_ms: int, candle_type: CandleType,
is_new_pair: bool = False, raise_: bool = False,
until_ms: int = None
) -> Tuple[str, str, str, List]:
"""
Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date
@ -115,7 +116,8 @@ class Binance(Exchange):
since_ms=since_ms,
is_new_pair=is_new_pair,
raise_=raise_,
candle_type=candle_type
candle_type=candle_type,
until_ms=until_ms,
)
def funding_fee_cutoff(self, open_date: datetime):

View File

@ -1645,7 +1645,8 @@ class Exchange:
def get_historic_ohlcv(self, pair: str, timeframe: str,
since_ms: int, candle_type: CandleType,
is_new_pair: bool = False) -> List:
is_new_pair: bool = False,
until_ms: int = None) -> List:
"""
Get candle history using asyncio and returns the list of candles.
Handles all async work for this.
@ -1653,13 +1654,14 @@ class Exchange:
:param pair: Pair to download
:param timeframe: Timeframe to get data for
:param since_ms: Timestamp in milliseconds to get history from
:param until_ms: Timestamp in milliseconds to get history up to
:param candle_type: '', mark, index, premiumIndex, or funding_rate
:return: List with candle (OHLCV) data
"""
pair, _, _, data = self.loop.run_until_complete(
self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe,
since_ms=since_ms, is_new_pair=is_new_pair,
candle_type=candle_type))
since_ms=since_ms, until_ms=until_ms,
is_new_pair=is_new_pair, candle_type=candle_type))
logger.info(f"Downloaded data for {pair} with length {len(data)}.")
return data
@ -1680,6 +1682,7 @@ class Exchange:
async def _async_get_historic_ohlcv(self, pair: str, timeframe: str,
since_ms: int, candle_type: CandleType,
is_new_pair: bool = False, raise_: bool = False,
until_ms: int = None
) -> Tuple[str, str, str, List]:
"""
Download historic ohlcv
@ -1695,7 +1698,7 @@ class Exchange:
)
input_coroutines = [self._async_get_candle_history(
pair, timeframe, candle_type, since) for since in
range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)]
range(since_ms, until_ms or (arrow.utcnow().int_timestamp * 1000), one_call)]
data: List = []
# Chunk requests into batches of 100 to avoid overwelming ccxt Throttling

View File

@ -402,7 +402,10 @@ class FreqtradeBot(LoggingMixin):
logger.info("No currency pair in active pair whitelist, "
"but checking to exit open trades.")
return trades_created
if PairLocks.is_global_lock():
if PairLocks.is_global_lock(side='*'):
# This only checks for total locks (both sides).
# per-side locks will be evaluated by `is_pair_locked` within create_trade,
# once the direction for the trade is clear.
lock = PairLocks.get_pair_longest_lock('*')
if lock:
self.log_once(f"Global pairlock active until "
@ -436,16 +439,6 @@ class FreqtradeBot(LoggingMixin):
analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(pair, self.strategy.timeframe)
nowtime = analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None
if self.strategy.is_pair_locked(pair, nowtime):
lock = PairLocks.get_pair_longest_lock(pair, nowtime)
if lock:
self.log_once(f"Pair {pair} is still locked until "
f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)} "
f"due to {lock.reason}.",
logger.info)
else:
self.log_once(f"Pair {pair} is still locked.", logger.info)
return False
# get_free_open_trades is checked before create_trade is called
# but it is still used here to prevent opening too many trades within one iteration
@ -461,6 +454,16 @@ class FreqtradeBot(LoggingMixin):
)
if signal:
if self.strategy.is_pair_locked(pair, candle_date=nowtime, side=signal):
lock = PairLocks.get_pair_longest_lock(pair, nowtime, signal)
if lock:
self.log_once(f"Pair {pair} {lock.side} is locked until "
f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)} "
f"due to {lock.reason}.",
logger.info)
else:
self.log_once(f"Pair {pair} is currently locked.", logger.info)
return False
stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge)
bid_check_dom = self.config.get('entry_pricing', {}).get('check_depth_of_market', {})
@ -1653,21 +1656,21 @@ class FreqtradeBot(LoggingMixin):
if not trade.is_open:
if send_msg and not stoploss_order and not trade.open_order_id:
self._notify_exit(trade, '', True)
self.handle_protections(trade.pair)
self.handle_protections(trade.pair, trade.trade_direction)
elif send_msg and not trade.open_order_id:
# Enter fill
self._notify_enter(trade, order, fill=True)
return False
def handle_protections(self, pair: str) -> None:
prot_trig = self.protections.stop_per_pair(pair)
def handle_protections(self, pair: str, side: LongShort) -> None:
prot_trig = self.protections.stop_per_pair(pair, side=side)
if prot_trig:
msg = {'type': RPCMessageType.PROTECTION_TRIGGER, }
msg.update(prot_trig.to_json())
self.rpc.send_msg(msg)
prot_trig_glb = self.protections.global_stop()
prot_trig_glb = self.protections.global_stop(side=side)
if prot_trig_glb:
msg = {'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, }
msg.update(prot_trig_glb.to_json())

View File

@ -867,10 +867,11 @@ class Backtesting:
return 'short'
return None
def run_protections(self, enable_protections, pair: str, current_time: datetime):
def run_protections(
self, enable_protections, pair: str, current_time: datetime, side: LongShort):
if enable_protections:
self.protections.stop_per_pair(pair, current_time)
self.protections.global_stop(current_time)
self.protections.stop_per_pair(pair, current_time, side)
self.protections.global_stop(current_time, side)
def manage_open_orders(self, trade: LocalTrade, current_time, row: Tuple) -> bool:
"""
@ -1030,7 +1031,7 @@ class Backtesting:
and self.trade_slot_available(max_open_trades, open_trade_count_start)
and current_time != end_date
and trade_dir is not None
and not PairLocks.is_pair_locked(pair, row[DATE_IDX])
and not PairLocks.is_pair_locked(pair, row[DATE_IDX], trade_dir)
):
trade = self._enter_trade(pair, row, trade_dir)
if trade:
@ -1068,7 +1069,8 @@ class Backtesting:
LocalTrade.close_bt_trade(trade)
trades.append(trade)
self.wallets.update()
self.run_protections(enable_protections, pair, current_time)
self.run_protections(
enable_protections, pair, current_time, trade.trade_direction)
# Move time one configured time_interval ahead.
self.progress.increment()
@ -1092,7 +1094,7 @@ class Backtesting:
timerange: TimeRange):
self.progress.init_step(BacktestState.ANALYZE, 0)
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
logger.info(f"Running backtesting for Strategy {strat.get_strategy_name()}")
backtest_start_time = datetime.now(timezone.utc)
self._set_strategy(strat)

View File

@ -10,7 +10,7 @@ from typing import Any, Dict
from pandas import DataFrame
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.data.metrics import calculate_max_drawdown
from freqtrade.optimize.hyperopt import IHyperOptLoss

View File

@ -8,7 +8,7 @@ from datetime import datetime
from pandas import DataFrame
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.data.metrics import calculate_max_drawdown
from freqtrade.optimize.hyperopt import IHyperOptLoss

View File

@ -9,7 +9,7 @@ individual needs.
"""
from pandas import DataFrame
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.data.metrics import calculate_max_drawdown
from freqtrade.optimize.hyperopt import IHyperOptLoss

View File

@ -9,8 +9,8 @@ from pandas import DataFrame, to_datetime
from tabulate import tabulate
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT
from freqtrade.data.btanalysis import (calculate_cagr, calculate_csum, calculate_market_change,
calculate_max_drawdown)
from freqtrade.data.metrics import (calculate_cagr, calculate_csum, calculate_market_change,
calculate_max_drawdown)
from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value
from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename

View File

@ -9,7 +9,7 @@ from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)
def get_table_names_for_table(inspector, tabletype):
def get_table_names_for_table(inspector, tabletype) -> List[str]:
return [t for t in inspector.get_table_names() if t.startswith(tabletype)]
@ -21,7 +21,7 @@ def get_column_def(columns: List, column: str, default: str) -> str:
return default if not has_column(columns, column) else column
def get_backup_name(tabs, backup_prefix: str):
def get_backup_name(tabs: List[str], backup_prefix: str):
table_back_name = backup_prefix
for i, table_back_name in enumerate(tabs):
table_back_name = f'{backup_prefix}{i}'
@ -56,6 +56,16 @@ def set_sequence_ids(engine, order_id, trade_id):
connection.execute(text(f"ALTER SEQUENCE trades_id_seq RESTART WITH {trade_id}"))
def drop_index_on_table(engine, inspector, table_bak_name):
with engine.begin() as connection:
# drop indexes on backup table in new session
for index in inspector.get_indexes(table_bak_name):
if engine.name == 'mysql':
connection.execute(text(f"drop index {index['name']} on {table_bak_name}"))
else:
connection.execute(text(f"drop index {index['name']}"))
def migrate_trades_and_orders_table(
decl_base, inspector, engine,
trade_back_name: str, cols: List,
@ -116,13 +126,7 @@ def migrate_trades_and_orders_table(
with engine.begin() as connection:
connection.execute(text(f"alter table trades rename to {trade_back_name}"))
with engine.begin() as connection:
# drop indexes on backup table in new session
for index in inspector.get_indexes(trade_back_name):
if engine.name == 'mysql':
connection.execute(text(f"drop index {index['name']} on {trade_back_name}"))
else:
connection.execute(text(f"drop index {index['name']}"))
drop_index_on_table(engine, inspector, trade_back_name)
order_id, trade_id = get_last_sequence_ids(engine, trade_back_name, order_back_name)
@ -205,6 +209,31 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List):
"""))
def migrate_pairlocks_table(
decl_base, inspector, engine,
pairlock_back_name: str, cols: List):
# Schema migration necessary
with engine.begin() as connection:
connection.execute(text(f"alter table pairlocks rename to {pairlock_back_name}"))
drop_index_on_table(engine, inspector, pairlock_back_name)
side = get_column_def(cols, 'side', "'*'")
# let SQLAlchemy create the schema as required
decl_base.metadata.create_all(engine)
# Copy data back - following the correct schema
with engine.begin() as connection:
connection.execute(text(f"""insert into pairlocks
(id, pair, side, reason, lock_time,
lock_end_time, active)
select id, pair, {side} side, reason, lock_time,
lock_end_time, active
from {pairlock_back_name}
"""))
def set_sqlite_to_wal(engine):
if engine.name == 'sqlite' and str(engine.url) != 'sqlite://':
# Set Mode to
@ -220,10 +249,13 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
cols_trades = inspector.get_columns('trades')
cols_orders = inspector.get_columns('orders')
cols_pairlocks = inspector.get_columns('pairlocks')
tabs = get_table_names_for_table(inspector, 'trades')
table_back_name = get_backup_name(tabs, 'trades_bak')
order_tabs = get_table_names_for_table(inspector, 'orders')
order_table_bak_name = get_backup_name(order_tabs, 'orders_bak')
pairlock_tabs = get_table_names_for_table(inspector, 'pairlocks')
pairlock_table_bak_name = get_backup_name(pairlock_tabs, 'pairlocks_bak')
# Check if migration necessary
# Migrates both trades and orders table!
@ -236,6 +268,13 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
decl_base, inspector, engine, table_back_name, cols_trades,
order_table_bak_name, cols_orders)
if not has_column(cols_pairlocks, 'side'):
logger.info(f"Running database migration for pairlocks - "
f"backup: {pairlock_table_bak_name}")
migrate_pairlocks_table(
decl_base, inspector, engine, pairlock_table_bak_name, cols_pairlocks
)
if 'orders' not in previous_tables and 'trades' in previous_tables:
raise OperationalException(
"Your database seems to be very old. "

View File

@ -7,13 +7,13 @@ from decimal import Decimal
from typing import Any, Dict, List, Optional
from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String,
create_engine, desc, func, inspect)
create_engine, desc, func, inspect, or_)
from sqlalchemy.exc import NoSuchModuleError
from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session, sessionmaker
from sqlalchemy.pool import StaticPool
from sqlalchemy.sql.schema import UniqueConstraint
from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES
from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES, LongShort
from freqtrade.enums import ExitType, TradingMode
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.leverage import interest
@ -393,7 +393,7 @@ class LocalTrade():
return "sell"
@property
def trade_direction(self) -> str:
def trade_direction(self) -> LongShort:
if self.is_short:
return "short"
else:
@ -1426,6 +1426,8 @@ class PairLock(_DECL_BASE):
id = Column(Integer, primary_key=True)
pair = Column(String(25), nullable=False, index=True)
# lock direction - long, short or * (for both)
side = Column(String(25), nullable=False, default="*")
reason = Column(String(255), nullable=True)
# Time the pair was locked (start time)
lock_time = Column(DateTime, nullable=False)
@ -1437,11 +1439,12 @@ class PairLock(_DECL_BASE):
def __repr__(self):
lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT)
lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT)
return (f'PairLock(id={self.id}, pair={self.pair}, lock_time={lock_time}, '
f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})')
return (
f'PairLock(id={self.id}, pair={self.pair}, side={self.side}, lock_time={lock_time}, '
f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})')
@staticmethod
def query_pair_locks(pair: Optional[str], now: datetime) -> Query:
def query_pair_locks(pair: Optional[str], now: datetime, side: str = '*') -> Query:
"""
Get all currently active locks for this pair
:param pair: Pair to check for. Returns all current locks if pair is empty
@ -1452,6 +1455,11 @@ class PairLock(_DECL_BASE):
PairLock.active.is_(True), ]
if pair:
filters.append(PairLock.pair == pair)
if side != '*':
filters.append(or_(PairLock.side == side, PairLock.side == '*'))
else:
filters.append(PairLock.side == '*')
return PairLock.query.filter(
*filters
)
@ -1466,5 +1474,6 @@ class PairLock(_DECL_BASE):
'lock_end_timestamp': int(self.lock_end_time.replace(tzinfo=timezone.utc
).timestamp() * 1000),
'reason': self.reason,
'side': self.side,
'active': self.active,
}

View File

@ -31,7 +31,7 @@ class PairLocks():
@staticmethod
def lock_pair(pair: str, until: datetime, reason: str = None, *,
now: datetime = None) -> PairLock:
now: datetime = None, side: str = '*') -> PairLock:
"""
Create PairLock from now to "until".
Uses database by default, unless PairLocks.use_db is set to False,
@ -40,12 +40,14 @@ class PairLocks():
:param until: End time of the lock. Will be rounded up to the next candle.
:param reason: Reason string that will be shown as reason for the lock
:param now: Current timestamp. Used to determine lock start time.
:param side: Side to lock pair, can be 'long', 'short' or '*'
"""
lock = PairLock(
pair=pair,
lock_time=now or datetime.now(timezone.utc),
lock_end_time=timeframe_to_next_date(PairLocks.timeframe, until),
reason=reason,
side=side,
active=True
)
if PairLocks.use_db:
@ -56,7 +58,8 @@ class PairLocks():
return lock
@staticmethod
def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]:
def get_pair_locks(
pair: Optional[str], now: Optional[datetime] = None, side: str = '*') -> List[PairLock]:
"""
Get all currently active locks for this pair
:param pair: Pair to check for. Returns all current locks if pair is empty
@ -67,26 +70,28 @@ class PairLocks():
now = datetime.now(timezone.utc)
if PairLocks.use_db:
return PairLock.query_pair_locks(pair, now).all()
return PairLock.query_pair_locks(pair, now, side).all()
else:
locks = [lock for lock in PairLocks.locks if (
lock.lock_end_time >= now
and lock.active is True
and (pair is None or lock.pair == pair)
and (lock.side == '*' or lock.side == side)
)]
return locks
@staticmethod
def get_pair_longest_lock(pair: str, now: Optional[datetime] = None) -> Optional[PairLock]:
def get_pair_longest_lock(
pair: str, now: Optional[datetime] = None, side: str = '*') -> Optional[PairLock]:
"""
Get the lock that expires the latest for the pair given.
"""
locks = PairLocks.get_pair_locks(pair, now)
locks = PairLocks.get_pair_locks(pair, now, side=side)
locks = sorted(locks, key=lambda l: l.lock_end_time, reverse=True)
return locks[0] if locks else None
@staticmethod
def unlock_pair(pair: str, now: Optional[datetime] = None) -> None:
def unlock_pair(pair: str, now: Optional[datetime] = None, side: str = '*') -> None:
"""
Release all locks for this pair.
:param pair: Pair to unlock
@ -97,7 +102,7 @@ class PairLocks():
now = datetime.now(timezone.utc)
logger.info(f"Releasing all locks for {pair}.")
locks = PairLocks.get_pair_locks(pair, now)
locks = PairLocks.get_pair_locks(pair, now, side=side)
for lock in locks:
lock.active = False
if PairLocks.use_db:
@ -134,7 +139,7 @@ class PairLocks():
lock.active = False
@staticmethod
def is_global_lock(now: Optional[datetime] = None) -> bool:
def is_global_lock(now: Optional[datetime] = None, side: str = '*') -> bool:
"""
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.now(timezone.utc)
@ -142,10 +147,10 @@ class PairLocks():
if not now:
now = datetime.now(timezone.utc)
return len(PairLocks.get_pair_locks('*', now)) > 0
return len(PairLocks.get_pair_locks('*', now, side)) > 0
@staticmethod
def is_pair_locked(pair: str, now: Optional[datetime] = None) -> bool:
def is_pair_locked(pair: str, now: Optional[datetime] = None, side: str = '*') -> bool:
"""
:param pair: Pair to check for
:param now: Datetime object (generated via datetime.now(timezone.utc)).
@ -154,7 +159,10 @@ class PairLocks():
if not now:
now = datetime.now(timezone.utc)
return len(PairLocks.get_pair_locks(pair, now)) > 0 or PairLocks.is_global_lock(now)
return (
len(PairLocks.get_pair_locks(pair, now, side)) > 0
or PairLocks.is_global_lock(now, side)
)
@staticmethod
def get_all_locks() -> List[PairLock]:

View File

@ -5,12 +5,13 @@ from typing import Any, Dict, List, Optional
import pandas as pd
from freqtrade.configuration import TimeRange
from freqtrade.data.btanalysis import (analyze_trade_parallelism, calculate_max_drawdown,
calculate_underwater, combine_dataframes_with_mean,
create_cum_profit, extract_trades_of_period, load_trades)
from freqtrade.data.btanalysis import (analyze_trade_parallelism, extract_trades_of_period,
load_trades)
from freqtrade.data.converter import trim_dataframe
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history import get_timerange, load_data
from freqtrade.data.metrics import (calculate_max_drawdown, calculate_underwater,
combine_dataframes_with_mean, create_cum_profit)
from freqtrade.enums import CandleType
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds

View File

@ -5,6 +5,7 @@ import logging
from datetime import datetime, timezone
from typing import Dict, List, Optional
from freqtrade.constants import LongShort
from freqtrade.persistence import PairLocks
from freqtrade.persistence.models import PairLock
from freqtrade.plugins.protections import IProtection
@ -44,28 +45,31 @@ class ProtectionManager():
"""
return [{p.name: p.short_desc()} for p in self._protection_handlers]
def global_stop(self, now: Optional[datetime] = None) -> Optional[PairLock]:
def global_stop(self, now: Optional[datetime] = None,
side: LongShort = 'long') -> Optional[PairLock]:
if not now:
now = datetime.now(timezone.utc)
result = None
for protection_handler in self._protection_handlers:
if protection_handler.has_global_stop:
lock, until, reason = protection_handler.global_stop(now)
# Early stopping - first positive result blocks further trades
if lock and until:
if not PairLocks.is_global_lock(until):
result = PairLocks.lock_pair('*', until, reason, now=now)
lock = protection_handler.global_stop(date_now=now, side=side)
if lock and lock.until:
if not PairLocks.is_global_lock(lock.until, side=lock.lock_side):
result = PairLocks.lock_pair(
'*', lock.until, lock.reason, now=now, side=lock.lock_side)
return result
def stop_per_pair(self, pair, now: Optional[datetime] = None) -> Optional[PairLock]:
def stop_per_pair(self, pair, now: Optional[datetime] = None,
side: LongShort = 'long') -> Optional[PairLock]:
if not now:
now = datetime.now(timezone.utc)
result = None
for protection_handler in self._protection_handlers:
if protection_handler.has_local_stop:
lock, until, reason = protection_handler.stop_per_pair(pair, now)
if lock and until:
if not PairLocks.is_pair_locked(pair, until):
result = PairLocks.lock_pair(pair, until, reason, now=now)
lock = protection_handler.stop_per_pair(
pair=pair, date_now=now, side=side)
if lock and lock.until:
if not PairLocks.is_pair_locked(pair, lock.until, lock.lock_side):
result = PairLocks.lock_pair(
pair, lock.until, lock.reason, now=now, side=lock.lock_side)
return result

View File

@ -1,7 +1,9 @@
import logging
from datetime import datetime, timedelta
from typing import Optional
from freqtrade.constants import LongShort
from freqtrade.persistence import Trade
from freqtrade.plugins.protections import IProtection, ProtectionReturn
@ -26,7 +28,7 @@ class CooldownPeriod(IProtection):
"""
return (f"{self.name} - Cooldown period of {self.stop_duration_str}.")
def _cooldown_period(self, pair: str, date_now: datetime, ) -> ProtectionReturn:
def _cooldown_period(self, pair: str, date_now: datetime) -> Optional[ProtectionReturn]:
"""
Get last trade for this pair
"""
@ -45,11 +47,15 @@ class CooldownPeriod(IProtection):
self.log_once(f"Cooldown for {pair} for {self.stop_duration_str}.", logger.info)
until = self.calculate_lock_end([trade], self._stop_duration)
return True, until, self._reason()
return ProtectionReturn(
lock=True,
until=until,
reason=self._reason(),
)
return False, None, None
return None
def global_stop(self, date_now: datetime) -> ProtectionReturn:
def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]:
"""
Stops trading (position entering) for all pairs
This must evaluate to true for the whole period of the "cooldown period".
@ -57,9 +63,10 @@ class CooldownPeriod(IProtection):
If true, all pairs will be locked with <reason> until <until>
"""
# Not implemented for cooldown period.
return False, None, None
return None
def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn:
def stop_per_pair(
self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]:
"""
Stops trading (position entering) for this pair
This must evaluate to true for the whole period of the "cooldown period".

View File

@ -1,9 +1,11 @@
import logging
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, List, Optional
from freqtrade.constants import LongShort
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import plural
from freqtrade.mixins import LoggingMixin
@ -12,7 +14,13 @@ from freqtrade.persistence import LocalTrade
logger = logging.getLogger(__name__)
ProtectionReturn = Tuple[bool, Optional[datetime], Optional[str]]
@dataclass
class ProtectionReturn:
lock: bool
until: datetime
reason: Optional[str]
lock_side: str = '*'
class IProtection(LoggingMixin, ABC):
@ -80,14 +88,15 @@ class IProtection(LoggingMixin, ABC):
"""
@abstractmethod
def global_stop(self, date_now: datetime) -> ProtectionReturn:
def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]:
"""
Stops trading (position entering) for all pairs
This must evaluate to true for the whole period of the "cooldown period".
"""
@abstractmethod
def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn:
def stop_per_pair(
self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]:
"""
Stops trading (position entering) for this pair
This must evaluate to true for the whole period of the "cooldown period".

View File

@ -1,8 +1,9 @@
import logging
from datetime import datetime, timedelta
from typing import Any, Dict
from typing import Any, Dict, Optional
from freqtrade.constants import LongShort
from freqtrade.persistence import Trade
from freqtrade.plugins.protections import IProtection, ProtectionReturn
@ -35,7 +36,7 @@ class LowProfitPairs(IProtection):
return (f'{profit} < {self._required_profit} in {self.lookback_period_str}, '
f'locking for {self.stop_duration_str}.')
def _low_profit(self, date_now: datetime, pair: str) -> ProtectionReturn:
def _low_profit(self, date_now: datetime, pair: str) -> Optional[ProtectionReturn]:
"""
Evaluate recent trades for pair
"""
@ -51,7 +52,7 @@ class LowProfitPairs(IProtection):
# trades = Trade.get_trades(filters).all()
if len(trades) < self._trade_limit:
# Not enough trades in the relevant period
return False, None, None
return None
profit = sum(trade.close_profit for trade in trades if trade.close_profit)
if profit < self._required_profit:
@ -60,20 +61,25 @@ class LowProfitPairs(IProtection):
f"within {self._lookback_period} minutes.", logger.info)
until = self.calculate_lock_end(trades, self._stop_duration)
return True, until, self._reason(profit)
return ProtectionReturn(
lock=True,
until=until,
reason=self._reason(profit),
)
return False, None, None
return None
def global_stop(self, date_now: datetime) -> ProtectionReturn:
def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]:
"""
Stops trading (position entering) for all pairs
This must evaluate to true for the whole period of the "cooldown period".
:return: Tuple of [bool, until, reason].
If true, all pairs will be locked with <reason> until <until>
"""
return False, None, None
return None
def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn:
def stop_per_pair(
self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]:
"""
Stops trading (position entering) for this pair
This must evaluate to true for the whole period of the "cooldown period".

View File

@ -1,11 +1,12 @@
import logging
from datetime import datetime, timedelta
from typing import Any, Dict
from typing import Any, Dict, Optional
import pandas as pd
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.constants import LongShort
from freqtrade.data.metrics import calculate_max_drawdown
from freqtrade.persistence import Trade
from freqtrade.plugins.protections import IProtection, ProtectionReturn
@ -39,7 +40,7 @@ class MaxDrawdown(IProtection):
return (f'{drawdown} passed {self._max_allowed_drawdown} in {self.lookback_period_str}, '
f'locking for {self.stop_duration_str}.')
def _max_drawdown(self, date_now: datetime) -> ProtectionReturn:
def _max_drawdown(self, date_now: datetime) -> Optional[ProtectionReturn]:
"""
Evaluate recent trades for drawdown ...
"""
@ -51,14 +52,14 @@ class MaxDrawdown(IProtection):
if len(trades) < self._trade_limit:
# Not enough trades in the relevant period
return False, None, None
return None
# Drawdown is always positive
try:
# TODO: This should use absolute profit calculation, considering account balance.
drawdown, _, _, _, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit')
except ValueError:
return False, None, None
return None
if drawdown > self._max_allowed_drawdown:
self.log_once(
@ -66,11 +67,15 @@ class MaxDrawdown(IProtection):
f" within {self.lookback_period_str}.", logger.info)
until = self.calculate_lock_end(trades, self._stop_duration)
return True, until, self._reason(drawdown)
return ProtectionReturn(
lock=True,
until=until,
reason=self._reason(drawdown),
)
return False, None, None
return None
def global_stop(self, date_now: datetime) -> ProtectionReturn:
def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]:
"""
Stops trading (position entering) for all pairs
This must evaluate to true for the whole period of the "cooldown period".
@ -79,11 +84,12 @@ class MaxDrawdown(IProtection):
"""
return self._max_drawdown(date_now)
def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn:
def stop_per_pair(
self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]:
"""
Stops trading (position entering) for this pair
This must evaluate to true for the whole period of the "cooldown period".
:return: Tuple of [bool, until, reason].
If true, this pair will be locked with <reason> until <until>
"""
return False, None, None
return None

View File

@ -1,8 +1,9 @@
import logging
from datetime import datetime, timedelta
from typing import Any, Dict
from typing import Any, Dict, Optional
from freqtrade.constants import LongShort
from freqtrade.enums import ExitType
from freqtrade.persistence import Trade
from freqtrade.plugins.protections import IProtection, ProtectionReturn
@ -21,6 +22,7 @@ class StoplossGuard(IProtection):
self._trade_limit = protection_config.get('trade_limit', 10)
self._disable_global_stop = protection_config.get('only_per_pair', False)
self._only_per_side = protection_config.get('only_per_side', False)
def short_desc(self) -> str:
"""
@ -36,7 +38,8 @@ class StoplossGuard(IProtection):
return (f'{self._trade_limit} stoplosses in {self._lookback_period} min, '
f'locking for {self._stop_duration} min.')
def _stoploss_guard(self, date_now: datetime, pair: str = None) -> ProtectionReturn:
def _stoploss_guard(
self, date_now: datetime, pair: Optional[str], side: str) -> Optional[ProtectionReturn]:
"""
Evaluate recent trades
"""
@ -48,15 +51,24 @@ class StoplossGuard(IProtection):
ExitType.STOPLOSS_ON_EXCHANGE.value)
and trade.close_profit and trade.close_profit < 0)]
if self._only_per_side:
# Long or short trades only
trades = [trade for trade in trades if trade.trade_direction == side]
if len(trades) < self._trade_limit:
return False, None, None
return None
self.log_once(f"Trading stopped due to {self._trade_limit} "
f"stoplosses within {self._lookback_period} minutes.", logger.info)
until = self.calculate_lock_end(trades, self._stop_duration)
return True, until, self._reason()
return ProtectionReturn(
lock=True,
until=until,
reason=self._reason(),
lock_side=(side if self._only_per_side else '*')
)
def global_stop(self, date_now: datetime) -> ProtectionReturn:
def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]:
"""
Stops trading (position entering) for all pairs
This must evaluate to true for the whole period of the "cooldown period".
@ -64,14 +76,15 @@ class StoplossGuard(IProtection):
If true, all pairs will be locked with <reason> until <until>
"""
if self._disable_global_stop:
return False, None, None
return self._stoploss_guard(date_now, None)
return None
return self._stoploss_guard(date_now, None, side)
def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn:
def stop_per_pair(
self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]:
"""
Stops trading (position entering) for this pair
This must evaluate to true for the whole period of the "cooldown period".
:return: Tuple of [bool, until, reason].
If true, this pair will be locked with <reason> until <until>
"""
return self._stoploss_guard(date_now, pair)
return self._stoploss_guard(date_now, pair, side)

View File

@ -23,7 +23,7 @@ class HyperOptLossResolver(IResolver):
object_type = IHyperOptLoss
object_type_str = "HyperoptLoss"
user_subdir = USERPATH_HYPEROPTS
initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve()
initial_search_path = Path(__file__).parent.parent.joinpath('optimize/hyperopt_loss').resolve()
@staticmethod
def load_hyperoptloss(config: Dict) -> IHyperOptLoss:

View File

@ -84,6 +84,7 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
lastconfig['enable_protections'] = btconfig.get('enable_protections')
lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet')
ApiServer._bt.strategylist = [strat]
ApiServer._bt.results = {}
ApiServer._bt.load_prior_backtest()

View File

@ -291,6 +291,7 @@ class LockModel(BaseModel):
lock_time: str
lock_timestamp: int
pair: str
side: str
reason: str

View File

@ -573,7 +573,7 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
return self.__class__.__name__
def lock_pair(self, pair: str, until: datetime, reason: str = None) -> None:
def lock_pair(self, pair: str, until: datetime, reason: str = None, side: str = '*') -> None:
"""
Locks pair until a given timestamp happens.
Locked pairs are not analyzed, and are prevented from opening new trades.
@ -583,8 +583,9 @@ class IStrategy(ABC, HyperStrategyMixin):
:param until: datetime in UTC until the pair should be blocked from opening new trades.
Needs to be timezone aware `datetime.now(timezone.utc)`
:param reason: Optional string explaining why the pair was locked.
:param side: Side to check, can be long, short or '*'
"""
PairLocks.lock_pair(pair, until, reason)
PairLocks.lock_pair(pair, until, reason, side=side)
def unlock_pair(self, pair: str) -> None:
"""
@ -604,7 +605,7 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
PairLocks.unlock_reason(reason, datetime.now(timezone.utc))
def is_pair_locked(self, pair: str, candle_date: datetime = None) -> bool:
def is_pair_locked(self, pair: str, *, candle_date: datetime = None, side: str = '*') -> bool:
"""
Checks if a pair is currently locked
The 2nd, optional parameter ensures that locks are applied until the new candle arrives,
@ -612,15 +613,16 @@ class IStrategy(ABC, HyperStrategyMixin):
of 2 seconds for an entry order to happen on an old signal.
:param pair: "Pair to check"
:param candle_date: Date of the last candle. Optional, defaults to current date
:param side: Side to check, can be long, short or '*'
:returns: locking state of the pair in question.
"""
if not candle_date:
# Simple call ...
return PairLocks.is_pair_locked(pair)
return PairLocks.is_pair_locked(pair, side=side)
else:
lock_time = timeframe_to_next_date(self.timeframe, candle_date)
return PairLocks.is_pair_locked(pair, lock_time)
return PairLocks.is_pair_locked(pair, lock_time, side=side)
def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""

View File

@ -8,14 +8,14 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime
from freqtrade.configuration import TimeRange
from freqtrade.constants import LAST_BT_RESULT_FN
from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, calculate_cagr,
calculate_csum, calculate_market_change,
calculate_max_drawdown, calculate_underwater,
combine_dataframes_with_mean, create_cum_profit,
from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism,
extract_trades_of_period, get_latest_backtest_filename,
get_latest_hyperopt_file, load_backtest_data,
load_backtest_metadata, load_trades, load_trades_from_db)
from freqtrade.data.history import load_data, load_pair_history
from freqtrade.data.metrics import (calculate_cagr, calculate_csum, calculate_market_change,
calculate_max_drawdown, calculate_underwater,
combine_dataframes_with_mean, create_cum_profit)
from freqtrade.exceptions import OperationalException
from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades
from tests.conftest_trades import MOCK_TRADE_COUNT

View File

@ -149,8 +149,8 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog,
load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC', candle_type=candle_type)
assert file.is_file()
assert log_has_re(
r'Download history data for pair: "MEME/BTC" \(0/1\), timeframe: 1m, '
r'candle type: spot and store in .*', caplog
r'\(0/1\) - Download history data for "MEME/BTC", 1m, '
r'spot and store in .*', caplog
)
@ -223,42 +223,65 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None:
# timeframe starts earlier than the cached data
# should fully update data
timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0)
data, start_ts = _load_cached_data_for_updating(
data, start_ts, end_ts = _load_cached_data_for_updating(
'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT)
assert data.empty
assert start_ts == test_data[0][0] - 1000
assert end_ts is None
# timeframe starts earlier than the cached data - prepending
timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0)
data, start_ts, end_ts = _load_cached_data_for_updating(
'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT, True)
assert_frame_equal(data, test_data_df.iloc[:-1])
assert start_ts == test_data[0][0] - 1000
assert end_ts == test_data[0][0]
# timeframe starts in the center of the cached data
# should return the cached data w/o the last item
timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0)
data, start_ts = _load_cached_data_for_updating(
data, start_ts, end_ts = _load_cached_data_for_updating(
'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT)
assert_frame_equal(data, test_data_df.iloc[:-1])
assert test_data[-2][0] <= start_ts < test_data[-1][0]
assert end_ts is None
# timeframe starts after the cached data
# should return the cached data w/o the last item
timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 100, 0)
data, start_ts = _load_cached_data_for_updating(
data, start_ts, end_ts = _load_cached_data_for_updating(
'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT)
assert_frame_equal(data, test_data_df.iloc[:-1])
assert test_data[-2][0] <= start_ts < test_data[-1][0]
assert end_ts is None
# no datafile exist
# should return timestamp start time
timerange = TimeRange('date', None, now_ts - 10000, 0)
data, start_ts = _load_cached_data_for_updating(
data, start_ts, end_ts = _load_cached_data_for_updating(
'NONEXIST/BTC', '1m', timerange, data_handler, CandleType.SPOT)
assert data.empty
assert start_ts == (now_ts - 10000) * 1000
assert end_ts is None
# no datafile exist
# should return timestamp start and end time time
timerange = TimeRange('date', 'date', now_ts - 1000000, now_ts - 100000)
data, start_ts, end_ts = _load_cached_data_for_updating(
'NONEXIST/BTC', '1m', timerange, data_handler, CandleType.SPOT)
assert data.empty
assert start_ts == (now_ts - 1000000) * 1000
assert end_ts == (now_ts - 100000) * 1000
# no datafile exist, no timeframe is set
# should return an empty array and None
data, start_ts = _load_cached_data_for_updating(
data, start_ts, end_ts = _load_cached_data_for_updating(
'NONEXIST/BTC', '1m', None, data_handler, CandleType.SPOT)
assert data.empty
assert start_ts is None
assert end_ts is None
@pytest.mark.parametrize('candle_type,subdir,file_tail', [

View File

@ -1983,6 +1983,20 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_
assert exchange._api_async.fetch_ohlcv.call_count > 200
assert res[0] == ohlcv[0]
exchange._api_async.fetch_ohlcv.reset_mock()
end_ts = 1_500_500_000_000
start_ts = 1_500_000_000_000
respair, restf, _, res = await exchange._async_get_historic_ohlcv(
pair, "5m", since_ms=start_ts, candle_type=candle_type, is_new_pair=False,
until_ms=end_ts
)
# Required candles
candles = (end_ts - start_ts) / 300_000
exp = candles // exchange.ohlcv_candle_limit('5m') + 1
# Depending on the exchange, this should be called between 1 and 6 times.
assert exchange._api_async.fetch_ohlcv.call_count == exp
@pytest.mark.parametrize('candle_type', [CandleType.FUTURES, CandleType.MARK, CandleType.SPOT])
def test_refresh_latest_ohlcv(mocker, default_conf, caplog, candle_type) -> None:

View File

@ -4,7 +4,7 @@ from unittest.mock import MagicMock
import pytest
from freqtrade.exceptions import OperationalException
from freqtrade.optimize.hyperopt_loss_short_trade_dur import ShortTradeDurHyperOptLoss
from freqtrade.optimize.hyperopt_loss.hyperopt_loss_short_trade_dur import ShortTradeDurHyperOptLoss
from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver

View File

@ -21,8 +21,22 @@ def test_PairLocks(use_db):
pair = 'ETH/BTC'
assert not PairLocks.is_pair_locked(pair)
PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
# ETH/BTC locked for 4 minutes
# ETH/BTC locked for 4 minutes (on both sides)
assert PairLocks.is_pair_locked(pair)
assert PairLocks.is_pair_locked(pair, side='long')
assert PairLocks.is_pair_locked(pair, side='short')
pair = 'BNB/BTC'
PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime, side='long')
assert not PairLocks.is_pair_locked(pair)
assert PairLocks.is_pair_locked(pair, side='long')
assert not PairLocks.is_pair_locked(pair, side='short')
pair = 'BNB/USDT'
PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime, side='short')
assert not PairLocks.is_pair_locked(pair)
assert not PairLocks.is_pair_locked(pair, side='long')
assert PairLocks.is_pair_locked(pair, side='short')
# XRP/BTC should not be locked now
pair = 'XRP/BTC'

View File

@ -11,9 +11,10 @@ from tests.conftest import get_patched_freqtradebot, log_has_re
def generate_mock_trade(pair: str, fee: float, is_open: bool,
sell_reason: str = ExitType.EXIT_SIGNAL,
exit_reason: str = ExitType.EXIT_SIGNAL,
min_ago_open: int = None, min_ago_close: int = None,
profit_rate: float = 0.9
profit_rate: float = 0.9,
is_short: bool = False,
):
open_rate = random.random()
@ -28,11 +29,12 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool,
is_open=is_open,
amount=0.01 / open_rate,
exchange='binance',
is_short=is_short,
)
trade.recalc_open_trade_value()
if not is_open:
trade.close(open_rate * profit_rate)
trade.exit_reason = sell_reason
trade.close(open_rate * (2 - profit_rate if is_short else profit_rate))
trade.exit_reason = exit_reason
return trade
@ -45,9 +47,9 @@ def test_protectionmanager(mocker, default_conf):
for handler in freqtrade.protections._protection_handlers:
assert handler.name in constants.AVAILABLE_PROTECTIONS
if not handler.has_global_stop:
assert handler.global_stop(datetime.utcnow()) == (False, None, None)
assert handler.global_stop(datetime.utcnow(), '*') is None
if not handler.has_local_stop:
assert handler.stop_per_pair('XRP/BTC', datetime.utcnow()) == (False, None, None)
assert handler.stop_per_pair('XRP/BTC', datetime.utcnow(), '*') is None
@pytest.mark.parametrize('timeframe,expected,protconf', [
@ -68,7 +70,7 @@ def test_protectionmanager(mocker, default_conf):
('1h', [60, 540],
[{"method": "StoplossGuard", "lookback_period_candles": 1, "stop_duration_candles": 9}]),
])
def test_protections_init(mocker, default_conf, timeframe, expected, protconf):
def test_protections_init(default_conf, timeframe, expected, protconf):
default_conf['timeframe'] = timeframe
man = ProtectionManager(default_conf, protconf)
assert len(man._protection_handlers) == len(protconf)
@ -76,8 +78,10 @@ def test_protections_init(mocker, default_conf, timeframe, expected, protconf):
assert man._protection_handlers[0]._stop_duration == expected[1]
@pytest.mark.parametrize('is_short', [False, True])
@pytest.mark.usefixtures("init_persistence")
def test_stoploss_guard(mocker, default_conf, fee, caplog):
def test_stoploss_guard(mocker, default_conf, fee, caplog, is_short):
# Active for both sides (long and short)
default_conf['protections'] = [{
"method": "StoplossGuard",
"lookback_period": 60,
@ -91,8 +95,8 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog):
caplog.clear()
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
min_ago_open=200, min_ago_close=30,
'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=200, min_ago_close=30, is_short=is_short,
))
assert not freqtrade.protections.global_stop()
@ -100,13 +104,13 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog):
caplog.clear()
# This trade does not count, as it's closed too long ago
Trade.query.session.add(generate_mock_trade(
'BCH/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
min_ago_open=250, min_ago_close=100,
'BCH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=250, min_ago_close=100, is_short=is_short,
))
Trade.query.session.add(generate_mock_trade(
'ETH/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
min_ago_open=240, min_ago_close=30,
'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=240, min_ago_close=30, is_short=is_short,
))
# 3 Trades closed - but the 2nd has been closed too long ago.
assert not freqtrade.protections.global_stop()
@ -114,8 +118,8 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog):
caplog.clear()
Trade.query.session.add(generate_mock_trade(
'LTC/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
min_ago_open=180, min_ago_close=30,
'LTC/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=180, min_ago_close=30, is_short=is_short,
))
assert freqtrade.protections.global_stop()
@ -130,15 +134,19 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog):
@pytest.mark.parametrize('only_per_pair', [False, True])
@pytest.mark.parametrize('only_per_side', [False, True])
@pytest.mark.usefixtures("init_persistence")
def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair):
def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair, only_per_side):
default_conf['protections'] = [{
"method": "StoplossGuard",
"lookback_period": 60,
"trade_limit": 2,
"stop_duration": 60,
"only_per_pair": only_per_pair
"only_per_pair": only_per_pair,
"only_per_side": only_per_side,
}]
check_side = 'long' if only_per_side else '*'
is_short = False
freqtrade = get_patched_freqtradebot(mocker, default_conf)
message = r"Trading stopped due to .*"
pair = 'XRP/BTC'
@ -148,8 +156,8 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair
caplog.clear()
Trade.query.session.add(generate_mock_trade(
pair, fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
min_ago_open=200, min_ago_close=30, profit_rate=0.9,
pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=200, min_ago_close=30, profit_rate=0.9, is_short=is_short
))
assert not freqtrade.protections.stop_per_pair(pair)
@ -158,13 +166,13 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair
caplog.clear()
# This trade does not count, as it's closed too long ago
Trade.query.session.add(generate_mock_trade(
pair, fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
min_ago_open=250, min_ago_close=100, profit_rate=0.9,
pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=250, min_ago_close=100, profit_rate=0.9, is_short=is_short
))
# Trade does not count for per pair stop as it's the wrong pair.
Trade.query.session.add(generate_mock_trade(
'ETH/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
min_ago_open=240, min_ago_close=30, profit_rate=0.9,
'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=240, min_ago_close=30, profit_rate=0.9, is_short=is_short
))
# 3 Trades closed - but the 2nd has been closed too long ago.
assert not freqtrade.protections.stop_per_pair(pair)
@ -176,16 +184,34 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair
caplog.clear()
# Trade does not count potentially, as it's in the wrong direction
Trade.query.session.add(generate_mock_trade(
pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=150, min_ago_close=25, profit_rate=0.9, is_short=not is_short
))
freqtrade.protections.stop_per_pair(pair)
assert freqtrade.protections.global_stop() != only_per_pair
assert PairLocks.is_pair_locked(pair, side=check_side) != (only_per_side and only_per_pair)
assert PairLocks.is_global_lock(side=check_side) != only_per_pair
if only_per_side:
assert not PairLocks.is_pair_locked(pair, side='*')
assert not PairLocks.is_global_lock(side='*')
caplog.clear()
# 2nd Trade that counts with correct pair
Trade.query.session.add(generate_mock_trade(
pair, fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
min_ago_open=180, min_ago_close=30, profit_rate=0.9,
pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=180, min_ago_close=30, profit_rate=0.9, is_short=is_short
))
freqtrade.protections.stop_per_pair(pair)
assert freqtrade.protections.global_stop() != only_per_pair
assert PairLocks.is_pair_locked(pair)
assert PairLocks.is_global_lock() != only_per_pair
assert PairLocks.is_pair_locked(pair, side=check_side)
assert PairLocks.is_global_lock(side=check_side) != only_per_pair
if only_per_side:
assert not PairLocks.is_pair_locked(pair, side='*')
assert not PairLocks.is_global_lock(side='*')
@pytest.mark.usefixtures("init_persistence")
@ -203,7 +229,7 @@ def test_CooldownPeriod(mocker, default_conf, fee, caplog):
caplog.clear()
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=200, min_ago_close=30,
))
@ -213,7 +239,7 @@ def test_CooldownPeriod(mocker, default_conf, fee, caplog):
assert not PairLocks.is_global_lock()
Trade.query.session.add(generate_mock_trade(
'ETH/BTC', fee.return_value, False, sell_reason=ExitType.ROI.value,
'ETH/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value,
min_ago_open=205, min_ago_close=35,
))
@ -242,7 +268,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog):
caplog.clear()
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=800, min_ago_close=450, profit_rate=0.9,
))
@ -253,7 +279,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog):
assert not PairLocks.is_global_lock()
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=200, min_ago_close=120, profit_rate=0.9,
))
@ -265,14 +291,14 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog):
# Add positive trade
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=ExitType.ROI.value,
'XRP/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value,
min_ago_open=20, min_ago_close=10, profit_rate=1.15,
))
assert not freqtrade.protections.stop_per_pair('XRP/BTC')
assert not PairLocks.is_pair_locked('XRP/BTC')
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=110, min_ago_close=20, profit_rate=0.8,
))
@ -300,15 +326,15 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog):
caplog.clear()
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=1000, min_ago_close=900, profit_rate=1.1,
))
Trade.query.session.add(generate_mock_trade(
'ETH/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=1000, min_ago_close=900, profit_rate=1.1,
))
Trade.query.session.add(generate_mock_trade(
'NEO/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
'NEO/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=1000, min_ago_close=900, profit_rate=1.1,
))
# No losing trade yet ... so max_drawdown will raise exception
@ -316,7 +342,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog):
assert not freqtrade.protections.stop_per_pair('XRP/BTC')
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=500, min_ago_close=400, profit_rate=0.9,
))
# Not locked with one trade
@ -326,7 +352,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog):
assert not PairLocks.is_global_lock()
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value,
'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value,
min_ago_open=1200, min_ago_close=1100, profit_rate=0.5,
))
@ -339,7 +365,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog):
# Winning trade ... (should not lock, does not change drawdown!)
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=ExitType.ROI.value,
'XRP/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value,
min_ago_open=320, min_ago_close=410, profit_rate=1.5,
))
assert not freqtrade.protections.global_stop()
@ -349,7 +375,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog):
# Add additional negative trade, causing a loss of > 15%
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=ExitType.ROI.value,
'XRP/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value,
min_ago_open=20, min_ago_close=10, profit_rate=0.8,
))
assert not freqtrade.protections.stop_per_pair('XRP/BTC')

View File

@ -1483,7 +1483,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir):
assert not result['running']
assert result['status_msg'] == 'Backtest reset'
ftbot.config['export'] = 'trades'
ftbot.config['backtest_cache'] = 'none'
ftbot.config['backtest_cache'] = 'day'
ftbot.config['user_data_dir'] = Path(tmpdir)
ftbot.config['exportfilename'] = Path(tmpdir) / "backtest_results"
ftbot.config['exportfilename'].mkdir()
@ -1556,19 +1556,19 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir):
ApiServer._bgtask_running = False
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy',
side_effect=DependencyException())
rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data))
assert log_has("Backtesting caused an error: ", caplog)
ftbot.config['backtest_cache'] = 'day'
# Rerun backtest (should get previous result)
rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data))
assert_response(rc)
result = rc.json()
assert log_has_re('Reusing result of previous backtest.*', caplog)
data['stake_amount'] = 101
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy',
side_effect=DependencyException())
rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data))
assert log_has("Backtesting caused an error: ", caplog)
# Delete backtesting to avoid leakage since the backtest-object may stick around.
rc = client_delete(client, f"{BASE_URI}/backtest")
assert_response(rc)

View File

@ -666,23 +666,23 @@ def test_is_pair_locked(default_conf):
assert not strategy.is_pair_locked(pair)
# latest candle is from 14:20, lock goes to 14:30
assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-10))
assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-50))
assert strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-10))
assert strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-50))
# latest candle is from 14:25 (lock should be lifted)
# Since this is the "new candle" available at 14:30
assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-4))
assert not strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-4))
# Should not be locked after time expired
assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=10))
assert not strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=10))
# Change timeframe to 15m
strategy.timeframe = '15m'
# Candle from 14:14 - lock goes until 14:30
assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-16))
assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-15, seconds=-2))
assert strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-16))
assert strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-15, seconds=-2))
# Candle from 14:15 - lock goes until 14:30
assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-15))
assert not strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-15))
def test_is_informative_pairs_callback(default_conf):

View File

@ -21,6 +21,7 @@ from freqtrade.exceptions import (DependencyException, ExchangeError, Insufficie
from freqtrade.freqtradebot import FreqtradeBot
from freqtrade.persistence import Order, PairLocks, Trade
from freqtrade.persistence.models import PairLock
from freqtrade.plugins.protections.iprotection import ProtectionReturn
from freqtrade.worker import Worker
from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker,
log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal,
@ -420,7 +421,7 @@ def test_enter_positions_global_pairlock(default_conf_usdt, ticker_usdt, limit_b
assert not log_has_re(message, caplog)
caplog.clear()
PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because')
PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because', side='*')
n = freqtrade.enter_positions()
assert n == 0
assert log_has_re(message, caplog)
@ -441,9 +442,9 @@ def test_handle_protections(mocker, default_conf_usdt, fee, is_short):
freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt)
freqtrade.protections._protection_handlers[1].global_stop = MagicMock(
return_value=(True, arrow.utcnow().shift(hours=1).datetime, "asdf"))
return_value=ProtectionReturn(True, arrow.utcnow().shift(hours=1).datetime, "asdf"))
create_mock_trades(fee, is_short)
freqtrade.handle_protections('ETC/BTC')
freqtrade.handle_protections('ETC/BTC', '*')
send_msg_mock = freqtrade.rpc.send_msg
assert send_msg_mock.call_count == 2
assert send_msg_mock.call_args_list[0][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER
@ -3793,13 +3794,16 @@ def test_locked_pairs(default_conf_usdt, ticker_usdt, fee,
exit_check=ExitCheckTuple(exit_type=ExitType.STOP_LOSS)
)
trade.close(ticker_usdt_sell_down()['bid'])
assert freqtrade.strategy.is_pair_locked(trade.pair)
assert freqtrade.strategy.is_pair_locked(trade.pair, side='*')
# Boths sides are locked
assert freqtrade.strategy.is_pair_locked(trade.pair, side='long')
assert freqtrade.strategy.is_pair_locked(trade.pair, side='short')
# reinit - should buy other pair.
caplog.clear()
freqtrade.enter_positions()
assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog)
assert log_has_re(fr"Pair {trade.pair} \* is locked.*", caplog)
@pytest.mark.parametrize("is_short", [False, True])

View File

@ -15,6 +15,7 @@ from freqtrade.enums import TradingMode
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.persistence import LocalTrade, Order, Trade, clean_dry_run_db, init_db
from freqtrade.persistence.migrations import get_last_sequence_ids, set_sequence_ids
from freqtrade.persistence.models import PairLock
from tests.conftest import create_mock_trades, create_mock_trades_with_leverage, log_has, log_has_re
@ -1427,6 +1428,55 @@ def test_migrate_set_sequence_ids():
assert engine.begin.call_count == 0
def test_migrate_pairlocks(mocker, default_conf, fee, caplog):
"""
Test Database migration (starting with new pairformat)
"""
caplog.set_level(logging.DEBUG)
# Always create all columns apart from the last!
create_table_old = """CREATE TABLE pairlocks (
id INTEGER NOT NULL,
pair VARCHAR(25) NOT NULL,
reason VARCHAR(255),
lock_time DATETIME NOT NULL,
lock_end_time DATETIME NOT NULL,
active BOOLEAN NOT NULL,
PRIMARY KEY (id)
)
"""
create_index1 = "CREATE INDEX ix_pairlocks_pair ON pairlocks (pair)"
create_index2 = "CREATE INDEX ix_pairlocks_lock_end_time ON pairlocks (lock_end_time)"
create_index3 = "CREATE INDEX ix_pairlocks_active ON pairlocks (active)"
insert_table_old = """INSERT INTO pairlocks (
id, pair, reason, lock_time, lock_end_time, active)
VALUES (1, 'ETH/BTC', 'Auto lock', '2021-07-12 18:41:03', '2021-07-11 18:45:00', 1)
"""
insert_table_old2 = """INSERT INTO pairlocks (
id, pair, reason, lock_time, lock_end_time, active)
VALUES (2, '*', 'Lock all', '2021-07-12 18:41:03', '2021-07-12 19:00:00', 1)
"""
engine = create_engine('sqlite://')
mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine)
# Create table using the old format
with engine.begin() as connection:
connection.execute(text(create_table_old))
connection.execute(text(insert_table_old))
connection.execute(text(insert_table_old2))
connection.execute(text(create_index1))
connection.execute(text(create_index2))
connection.execute(text(create_index3))
init_db(default_conf['db_url'], default_conf['dry_run'])
assert len(PairLock.query.all()) == 2
assert len(PairLock.query.filter(PairLock.pair == '*').all()) == 1
pairlocks = PairLock.query.filter(PairLock.pair == 'ETH/BTC').all()
assert len(pairlocks) == 1
pairlocks[0].pair == 'ETH/BTC'
pairlocks[0].side == '*'
def test_adjust_stop_loss(fee):
trade = Trade(
pair='ADA/USDT',

View File

@ -10,7 +10,8 @@ from plotly.subplots import make_subplots
from freqtrade.commands import start_plot_dataframe, start_plot_profit
from freqtrade.configuration import TimeRange
from freqtrade.data import history
from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data
from freqtrade.data.btanalysis import load_backtest_data
from freqtrade.data.metrics import create_cum_profit
from freqtrade.exceptions import OperationalException
from freqtrade.plot.plotting import (add_areas, add_indicators, add_profit, create_plotconfig,
generate_candlestick_graph, generate_plot_filename,