diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7c0655b20..90a10d4da 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,9 +1,9 @@ -Thank you for sending your pull request. But first, have you included + ## Summary -Explain in one sentence the goal of this PR + Solve the issue: #___ @@ -14,4 +14,4 @@ Solve the issue: #___ ## What's new? -*Explain in details what this PR solve or improve. You can include visuals.* + diff --git a/docs/data-download.md b/docs/data-download.md index 9bfc1e685..681fb717d 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -30,6 +30,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--data-format-ohlcv {json,jsongz,hdf5}] [--data-format-trades {json,jsongz,hdf5}] [--trading-mode {spot,margin,futures}] + [--prepend] optional arguments: -h, --help show this help message and exit @@ -62,6 +63,7 @@ optional arguments: `jsongz`). --trading-mode {spot,margin,futures} Select Trading mode + --prepend Allow data prepending. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -157,10 +159,21 @@ freqtrade download-data --exchange binance --pairs .*/USDT - To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.) - To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. - To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days). -- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. Eventually set end dates are ignored. +- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. - Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data. - To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. +#### Download additional data before the current timerange + +Assuming you downloaded all data from 2022 (`--timerange 20220101-`) - but you'd now like to also backtest with earlier data. +You can do so by using the `--prepend` flag, combined with `--timerange` - specifying an end-date. + +``` bash +freqtrade download-data --exchange binance --pairs ETH/USDT XRP/USDT BTC/USDT --prepend --timerange 20210101-20220101 +``` + +!!! Note + Freqtrade will ignore the end-date in this mode if data is available, updating the end-date to the existing data start point. ### Data format diff --git a/docs/developer.md b/docs/developer.md index 1cc16294b..185bfc92e 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -200,11 +200,12 @@ For that reason, they must implement the following methods: * `global_stop()` * `stop_per_pair()`. -`global_stop()` and `stop_per_pair()` must return a ProtectionReturn tuple, which consists of: +`global_stop()` and `stop_per_pair()` must return a ProtectionReturn object, which consists of: * lock pair - boolean * lock until - datetime - until when should the pair be locked (will be rounded up to the next new candle) * reason - string, used for logging and storage in the database +* lock_side - long, short or '*'. The `until` portion should be calculated using the provided `calculate_lock_end()` method. diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 0757d2f6d..bb4a7eb35 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -48,6 +48,8 @@ If `trade_limit` or more trades resulted in stoploss, trading will stop for `sto This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time. +Similarly, this protection will by default look at all trades (long and short). For futures bots, setting `only_per_side` will make the bot only consider one side, and will then only lock this one side, allowing for example shorts to continue after a series of long stoplosses. + The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. ``` python @@ -59,7 +61,8 @@ def protections(self): "lookback_period_candles": 24, "trade_limit": 4, "stop_duration_candles": 4, - "only_per_pair": False + "only_per_pair": False, + "only_per_side": False } ] ``` diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 62b79da2e..ff1d16590 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -72,7 +72,8 @@ ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive", "timerange", "download_trades", "exchange", "timeframes", - "erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode"] + "erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode", + "prepend_data"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index df8966e85..58e208652 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -443,6 +443,11 @@ AVAILABLE_CLI_OPTIONS = { default=['1m', '5m'], nargs='+', ), + "prepend_data": Arg( + '--prepend', + help='Allow data prepending.', + action='store_true', + ), "erase": Arg( '--erase', help='Clean all existing data for the selected exchange/pairs/timeframes.', diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index e41512ccc..a2e2a100a 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -85,6 +85,7 @@ def start_download_data(args: Dict[str, Any]) -> None: new_pairs_days=config['new_pairs_days'], erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'], trading_mode=config.get('trading_mode', 'spot'), + prepend=config.get('prepend_data', False) ) except KeyboardInterrupt: diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py index 02f2d4089..d49bf61f6 100644 --- a/freqtrade/configuration/config_setup.py +++ b/freqtrade/configuration/config_setup.py @@ -22,6 +22,6 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str # Ensure these modes are using Dry-run config['dry_run'] = True - validate_config_consistency(config) + validate_config_consistency(config, preliminary=True) return config diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 6e4a4b0ef..ee846e7e6 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -39,7 +39,7 @@ def _extend_validator(validator_class): FreqtradeValidator = _extend_validator(Draft4Validator) -def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: +def validate_config_schema(conf: Dict[str, Any], preliminary: bool = False) -> Dict[str, Any]: """ Validate the configuration follow the Config Schema :param conf: Config in JSON format @@ -49,7 +49,10 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: if conf.get('runmode', RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE): conf_schema['required'] = constants.SCHEMA_TRADE_REQUIRED elif conf.get('runmode', RunMode.OTHER) in (RunMode.BACKTEST, RunMode.HYPEROPT): - conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED + if preliminary: + conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED + else: + conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED_FINAL else: conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED try: @@ -64,7 +67,7 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: ) -def validate_config_consistency(conf: Dict[str, Any]) -> None: +def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False) -> None: """ Validate the configuration consistency. Should be ran after loading both configuration and strategy, @@ -85,7 +88,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None: # validate configuration before returning logger.info('Validating configuration ...') - validate_config_schema(conf) + validate_config_schema(conf, preliminary=preliminary) def _validate_unlimited_amount(conf: Dict[str, Any]) -> None: diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index dde56c220..80df6fb3f 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -393,6 +393,8 @@ class Configuration: self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') + self._args_to_config(config, argname='prepend_data', + logstring='Prepend detected. Allowing data prepending.') self._args_to_config(config, argname='erase', logstring='Erase detected. Deleting existing data.') diff --git a/freqtrade/constants.py b/freqtrade/constants.py index af5ad00dd..8f0ff5ead 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -462,6 +462,10 @@ SCHEMA_BACKTEST_REQUIRED = [ 'dataformat_ohlcv', 'dataformat_trades', ] +SCHEMA_BACKTEST_REQUIRED_FINAL = SCHEMA_BACKTEST_REQUIRED + [ + 'stoploss', + 'minimal_roi', +] SCHEMA_MINIMAL_REQUIRED = [ 'exchange', diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 0c8e721c0..e29d9ebe4 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -5,7 +5,7 @@ import logging from copy import copy from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Union import numpy as np import pandas as pd @@ -400,168 +400,3 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame, trades = trades.loc[(trades['open_date'] >= trades_start) & (trades['close_date'] <= trades_stop)] return trades - - -def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float: - """ - Calculate market change based on "column". - Calculation is done by taking the first non-null and the last non-null element of each column - and calculating the pctchange as "(last - first) / first". - Then the results per pair are combined as mean. - - :param data: Dict of Dataframes, dict key should be pair. - :param column: Column in the original dataframes to use - :return: - """ - tmp_means = [] - for pair, df in data.items(): - start = df[column].dropna().iloc[0] - end = df[column].dropna().iloc[-1] - tmp_means.append((end - start) / start) - - return float(np.mean(tmp_means)) - - -def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame], - column: str = "close") -> pd.DataFrame: - """ - Combine multiple dataframes "column" - :param data: Dict of Dataframes, dict key should be pair. - :param column: Column in the original dataframes to use - :return: DataFrame with the column renamed to the dict key, and a column - named mean, containing the mean of all pairs. - :raise: ValueError if no data is provided. - """ - df_comb = pd.concat([data[pair].set_index('date').rename( - {column: pair}, axis=1)[pair] for pair in data], axis=1) - - df_comb['mean'] = df_comb.mean(axis=1) - - return df_comb - - -def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, - timeframe: str) -> pd.DataFrame: - """ - Adds a column `col_name` with the cumulative profit for the given trades array. - :param df: DataFrame with date index - :param trades: DataFrame containing trades (requires columns close_date and profit_abs) - :param col_name: Column name that will be assigned the results - :param timeframe: Timeframe used during the operations - :return: Returns df with one additional column, col_name, containing the cumulative profit. - :raise: ValueError if trade-dataframe was found empty. - """ - if len(trades) == 0: - raise ValueError("Trade dataframe empty.") - from freqtrade.exchange import timeframe_to_minutes - timeframe_minutes = timeframe_to_minutes(timeframe) - # Resample to timeframe to make sure trades match candles - _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date' - )[['profit_abs']].sum() - df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum() - # Set first value to 0 - df.loc[df.iloc[0].name, col_name] = 0 - # FFill to get continuous - df[col_name] = df[col_name].ffill() - return df - - -def _calc_drawdown_series(profit_results: pd.DataFrame, *, date_col: str, value_col: str - ) -> pd.DataFrame: - max_drawdown_df = pd.DataFrame() - max_drawdown_df['cumulative'] = profit_results[value_col].cumsum() - max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax() - max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value'] - max_drawdown_df['date'] = profit_results.loc[:, date_col] - return max_drawdown_df - - -def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date', - value_col: str = 'profit_ratio' - ): - """ - Calculate max drawdown and the corresponding close dates - :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) - :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') - :param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio') - :return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown, - high and low time and high and low value. - :raise: ValueError if trade-dataframe was found empty. - """ - if len(trades) == 0: - raise ValueError("Trade dataframe empty.") - profit_results = trades.sort_values(date_col).reset_index(drop=True) - max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) - - return max_drawdown_df - - -def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date', - value_col: str = 'profit_abs', starting_balance: float = 0 - ) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float, float]: - """ - Calculate max drawdown and the corresponding close dates - :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) - :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') - :param value_col: Column in DataFrame to use for values (defaults to 'profit_abs') - :param starting_balance: Portfolio starting balance - properly calculate relative drawdown. - :return: Tuple (float, highdate, lowdate, highvalue, lowvalue, relative_drawdown) - with absolute max drawdown, high and low time and high and low value, - and the relative account drawdown - :raise: ValueError if trade-dataframe was found empty. - """ - if len(trades) == 0: - raise ValueError("Trade dataframe empty.") - profit_results = trades.sort_values(date_col).reset_index(drop=True) - max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) - - idxmin = max_drawdown_df['drawdown'].idxmin() - if idxmin == 0: - raise ValueError("No losing trade, therefore no drawdown.") - high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col] - low_date = profit_results.loc[idxmin, date_col] - high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin] - ['high_value'].idxmax(), 'cumulative'] - low_val = max_drawdown_df.loc[idxmin, 'cumulative'] - max_drawdown_rel = 0.0 - if high_val + starting_balance != 0: - max_drawdown_rel = (high_val - low_val) / (high_val + starting_balance) - - return ( - abs(min(max_drawdown_df['drawdown'])), - high_date, - low_date, - high_val, - low_val, - max_drawdown_rel - ) - - -def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]: - """ - Calculate min/max cumsum of trades, to show if the wallet/stake amount ratio is sane - :param trades: DataFrame containing trades (requires columns close_date and profit_percent) - :param starting_balance: Add starting balance to results, to show the wallets high / low points - :return: Tuple (float, float) with cumsum of profit_abs - :raise: ValueError if trade-dataframe was found empty. - """ - if len(trades) == 0: - raise ValueError("Trade dataframe empty.") - - csum_df = pd.DataFrame() - csum_df['sum'] = trades['profit_abs'].cumsum() - csum_min = csum_df['sum'].min() + starting_balance - csum_max = csum_df['sum'].max() + starting_balance - - return csum_min, csum_max - - -def calculate_cagr(days_passed: int, starting_balance: float, final_balance: float) -> float: - """ - Calculate CAGR - :param days_passed: Days passed between start and ending balance - :param starting_balance: Starting balance - :param final_balance: Final balance to calculate CAGR against - :return: CAGR - """ - return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1 diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 8560fd29e..eb36d2042 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -139,8 +139,9 @@ def _load_cached_data_for_updating( timeframe: str, timerange: Optional[TimeRange], data_handler: IDataHandler, - candle_type: CandleType -) -> Tuple[DataFrame, Optional[int]]: + candle_type: CandleType, + prepend: bool = False, +) -> Tuple[DataFrame, Optional[int], Optional[int]]: """ Load cached data to download more data. If timerange is passed in, checks whether data from an before the stored data will be @@ -150,9 +151,12 @@ def _load_cached_data_for_updating( Note: Only used by download_pair_history(). """ start = None + end = None if timerange: if timerange.starttype == 'date': start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + if timerange.stoptype == 'date': + end = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) # Intentionally don't pass timerange in - since we need to load the full dataset. data = data_handler.ohlcv_load(pair, timeframe=timeframe, @@ -160,14 +164,17 @@ def _load_cached_data_for_updating( drop_incomplete=True, warn_no_data=False, candle_type=candle_type) if not data.empty: - if start and start < data.iloc[0]['date']: + if not prepend and start and start < data.iloc[0]['date']: # Earlier data than existing data requested, redownload all data = DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS) else: - start = data.iloc[-1]['date'] - + if prepend: + end = data.iloc[0]['date'] + else: + start = data.iloc[-1]['date'] start_ms = int(start.timestamp() * 1000) if start else None - return data, start_ms + end_ms = int(end.timestamp() * 1000) if end else None + return data, start_ms, end_ms def _download_pair_history(pair: str, *, @@ -180,6 +187,7 @@ def _download_pair_history(pair: str, *, timerange: Optional[TimeRange] = None, candle_type: CandleType, erase: bool = False, + prepend: bool = False, ) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters @@ -187,8 +195,6 @@ def _download_pair_history(pair: str, *, exists in a cache. If timerange starts earlier than the data in the cache, the full data will be redownloaded - Based on @Rybolov work: https://github.com/rybolov/freqtrade-data - :param pair: pair to download :param timeframe: Timeframe (e.g "5m") :param timerange: range of time to download @@ -203,14 +209,17 @@ def _download_pair_history(pair: str, *, if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): logger.info(f'Deleting existing data for pair {pair}, {timeframe}, {candle_type}.') - logger.info( - f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe}, ' - f'candle type: {candle_type} and store in {datadir}.' - ) + data, since_ms, until_ms = _load_cached_data_for_updating( + pair, timeframe, timerange, + data_handler=data_handler, + candle_type=candle_type, + prepend=prepend) - data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange, - data_handler=data_handler, - candle_type=candle_type) + logger.info(f'({process}) - Download history data for "{pair}", {timeframe}, ' + f'{candle_type} and store in {datadir}.' + f'From {format_ms_time(since_ms) if since_ms else "start"} to ' + f'{format_ms_time(until_ms) if until_ms else "now"}' + ) logger.debug("Current Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') @@ -225,6 +234,7 @@ def _download_pair_history(pair: str, *, days=-new_pairs_days).int_timestamp * 1000, is_new_pair=data.empty, candle_type=candle_type, + until_ms=until_ms if until_ms else None ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, @@ -257,6 +267,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes timerange: Optional[TimeRange] = None, new_pairs_days: int = 30, erase: bool = False, data_format: str = None, + prepend: bool = False, ) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. @@ -280,7 +291,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes timerange=timerange, data_handler=data_handler, timeframe=str(timeframe), new_pairs_days=new_pairs_days, candle_type=candle_type, - erase=erase) + erase=erase, prepend=prepend) if trading_mode == 'futures': # Predefined candletype (and timeframe) depending on exchange # Downloads what is necessary to backtest based on futures data. @@ -294,7 +305,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes timerange=timerange, data_handler=data_handler, timeframe=str(tf_mark), new_pairs_days=new_pairs_days, candle_type=funding_candle_type, - erase=erase) + erase=erase, prepend=prepend) return pairs_not_available @@ -312,8 +323,9 @@ def _download_trades_history(exchange: Exchange, try: until = None - if (timerange and timerange.starttype == 'date'): - since = timerange.startts * 1000 + if timerange: + if timerange.starttype == 'date': + since = timerange.startts * 1000 if timerange.stoptype == 'date': until = timerange.stopts * 1000 else: diff --git a/freqtrade/data/metrics.py b/freqtrade/data/metrics.py new file mode 100644 index 000000000..44d5ce6ec --- /dev/null +++ b/freqtrade/data/metrics.py @@ -0,0 +1,173 @@ +import logging +from typing import Dict, Tuple + +import numpy as np +import pandas as pd + + +logger = logging.getLogger(__name__) + + +def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float: + """ + Calculate market change based on "column". + Calculation is done by taking the first non-null and the last non-null element of each column + and calculating the pctchange as "(last - first) / first". + Then the results per pair are combined as mean. + + :param data: Dict of Dataframes, dict key should be pair. + :param column: Column in the original dataframes to use + :return: + """ + tmp_means = [] + for pair, df in data.items(): + start = df[column].dropna().iloc[0] + end = df[column].dropna().iloc[-1] + tmp_means.append((end - start) / start) + + return float(np.mean(tmp_means)) + + +def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame], + column: str = "close") -> pd.DataFrame: + """ + Combine multiple dataframes "column" + :param data: Dict of Dataframes, dict key should be pair. + :param column: Column in the original dataframes to use + :return: DataFrame with the column renamed to the dict key, and a column + named mean, containing the mean of all pairs. + :raise: ValueError if no data is provided. + """ + df_comb = pd.concat([data[pair].set_index('date').rename( + {column: pair}, axis=1)[pair] for pair in data], axis=1) + + df_comb['mean'] = df_comb.mean(axis=1) + + return df_comb + + +def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, + timeframe: str) -> pd.DataFrame: + """ + Adds a column `col_name` with the cumulative profit for the given trades array. + :param df: DataFrame with date index + :param trades: DataFrame containing trades (requires columns close_date and profit_abs) + :param col_name: Column name that will be assigned the results + :param timeframe: Timeframe used during the operations + :return: Returns df with one additional column, col_name, containing the cumulative profit. + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + from freqtrade.exchange import timeframe_to_minutes + timeframe_minutes = timeframe_to_minutes(timeframe) + # Resample to timeframe to make sure trades match candles + _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date' + )[['profit_abs']].sum() + df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum() + # Set first value to 0 + df.loc[df.iloc[0].name, col_name] = 0 + # FFill to get continuous + df[col_name] = df[col_name].ffill() + return df + + +def _calc_drawdown_series(profit_results: pd.DataFrame, *, date_col: str, value_col: str + ) -> pd.DataFrame: + max_drawdown_df = pd.DataFrame() + max_drawdown_df['cumulative'] = profit_results[value_col].cumsum() + max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax() + max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value'] + max_drawdown_df['date'] = profit_results.loc[:, date_col] + return max_drawdown_df + + +def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date', + value_col: str = 'profit_ratio' + ): + """ + Calculate max drawdown and the corresponding close dates + :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) + :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') + :param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio') + :return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown, + high and low time and high and low value. + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + profit_results = trades.sort_values(date_col).reset_index(drop=True) + max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) + + return max_drawdown_df + + +def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date', + value_col: str = 'profit_abs', starting_balance: float = 0 + ) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float, float]: + """ + Calculate max drawdown and the corresponding close dates + :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) + :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') + :param value_col: Column in DataFrame to use for values (defaults to 'profit_abs') + :param starting_balance: Portfolio starting balance - properly calculate relative drawdown. + :return: Tuple (float, highdate, lowdate, highvalue, lowvalue, relative_drawdown) + with absolute max drawdown, high and low time and high and low value, + and the relative account drawdown + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + profit_results = trades.sort_values(date_col).reset_index(drop=True) + max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) + + idxmin = max_drawdown_df['drawdown'].idxmin() + if idxmin == 0: + raise ValueError("No losing trade, therefore no drawdown.") + high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col] + low_date = profit_results.loc[idxmin, date_col] + high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin] + ['high_value'].idxmax(), 'cumulative'] + low_val = max_drawdown_df.loc[idxmin, 'cumulative'] + max_drawdown_rel = 0.0 + if high_val + starting_balance != 0: + max_drawdown_rel = (high_val - low_val) / (high_val + starting_balance) + + return ( + abs(min(max_drawdown_df['drawdown'])), + high_date, + low_date, + high_val, + low_val, + max_drawdown_rel + ) + + +def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]: + """ + Calculate min/max cumsum of trades, to show if the wallet/stake amount ratio is sane + :param trades: DataFrame containing trades (requires columns close_date and profit_percent) + :param starting_balance: Add starting balance to results, to show the wallets high / low points + :return: Tuple (float, float) with cumsum of profit_abs + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + + csum_df = pd.DataFrame() + csum_df['sum'] = trades['profit_abs'].cumsum() + csum_min = csum_df['sum'].min() + starting_balance + csum_max = csum_df['sum'].max() + starting_balance + + return csum_min, csum_max + + +def calculate_cagr(days_passed: int, starting_balance: float, final_balance: float) -> float: + """ + Calculate CAGR + :param days_passed: Days passed between start and ending balance + :param starting_balance: Starting balance + :param final_balance: Final balance to calculate CAGR against + :return: CAGR + """ + return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1 diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 8c442cd26..69ae5198a 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -95,6 +95,7 @@ class Binance(Exchange): async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int, candle_type: CandleType, is_new_pair: bool = False, raise_: bool = False, + until_ms: int = None ) -> Tuple[str, str, str, List]: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date @@ -115,7 +116,8 @@ class Binance(Exchange): since_ms=since_ms, is_new_pair=is_new_pair, raise_=raise_, - candle_type=candle_type + candle_type=candle_type, + until_ms=until_ms, ) def funding_fee_cutoff(self, open_date: datetime): diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 82dcacb51..2ed10ee7a 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1645,7 +1645,8 @@ class Exchange: def get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int, candle_type: CandleType, - is_new_pair: bool = False) -> List: + is_new_pair: bool = False, + until_ms: int = None) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1653,13 +1654,14 @@ class Exchange: :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from + :param until_ms: Timestamp in milliseconds to get history up to :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: List with candle (OHLCV) data """ pair, _, _, data = self.loop.run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms, is_new_pair=is_new_pair, - candle_type=candle_type)) + since_ms=since_ms, until_ms=until_ms, + is_new_pair=is_new_pair, candle_type=candle_type)) logger.info(f"Downloaded data for {pair} with length {len(data)}.") return data @@ -1680,6 +1682,7 @@ class Exchange: async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int, candle_type: CandleType, is_new_pair: bool = False, raise_: bool = False, + until_ms: int = None ) -> Tuple[str, str, str, List]: """ Download historic ohlcv @@ -1695,7 +1698,7 @@ class Exchange: ) input_coroutines = [self._async_get_candle_history( pair, timeframe, candle_type, since) for since in - range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] + range(since_ms, until_ms or (arrow.utcnow().int_timestamp * 1000), one_call)] data: List = [] # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5a16ef562..9258b7ca2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -402,7 +402,10 @@ class FreqtradeBot(LoggingMixin): logger.info("No currency pair in active pair whitelist, " "but checking to exit open trades.") return trades_created - if PairLocks.is_global_lock(): + if PairLocks.is_global_lock(side='*'): + # This only checks for total locks (both sides). + # per-side locks will be evaluated by `is_pair_locked` within create_trade, + # once the direction for the trade is clear. lock = PairLocks.get_pair_longest_lock('*') if lock: self.log_once(f"Global pairlock active until " @@ -436,16 +439,6 @@ class FreqtradeBot(LoggingMixin): analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(pair, self.strategy.timeframe) nowtime = analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None - if self.strategy.is_pair_locked(pair, nowtime): - lock = PairLocks.get_pair_longest_lock(pair, nowtime) - if lock: - self.log_once(f"Pair {pair} is still locked until " - f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)} " - f"due to {lock.reason}.", - logger.info) - else: - self.log_once(f"Pair {pair} is still locked.", logger.info) - return False # get_free_open_trades is checked before create_trade is called # but it is still used here to prevent opening too many trades within one iteration @@ -461,6 +454,16 @@ class FreqtradeBot(LoggingMixin): ) if signal: + if self.strategy.is_pair_locked(pair, candle_date=nowtime, side=signal): + lock = PairLocks.get_pair_longest_lock(pair, nowtime, signal) + if lock: + self.log_once(f"Pair {pair} {lock.side} is locked until " + f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)} " + f"due to {lock.reason}.", + logger.info) + else: + self.log_once(f"Pair {pair} is currently locked.", logger.info) + return False stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge) bid_check_dom = self.config.get('entry_pricing', {}).get('check_depth_of_market', {}) @@ -1653,21 +1656,21 @@ class FreqtradeBot(LoggingMixin): if not trade.is_open: if send_msg and not stoploss_order and not trade.open_order_id: self._notify_exit(trade, '', True) - self.handle_protections(trade.pair) + self.handle_protections(trade.pair, trade.trade_direction) elif send_msg and not trade.open_order_id: # Enter fill self._notify_enter(trade, order, fill=True) return False - def handle_protections(self, pair: str) -> None: - prot_trig = self.protections.stop_per_pair(pair) + def handle_protections(self, pair: str, side: LongShort) -> None: + prot_trig = self.protections.stop_per_pair(pair, side=side) if prot_trig: msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } msg.update(prot_trig.to_json()) self.rpc.send_msg(msg) - prot_trig_glb = self.protections.global_stop() + prot_trig_glb = self.protections.global_stop(side=side) if prot_trig_glb: msg = {'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, } msg.update(prot_trig_glb.to_json()) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d54c0e5a9..376236747 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -867,10 +867,11 @@ class Backtesting: return 'short' return None - def run_protections(self, enable_protections, pair: str, current_time: datetime): + def run_protections( + self, enable_protections, pair: str, current_time: datetime, side: LongShort): if enable_protections: - self.protections.stop_per_pair(pair, current_time) - self.protections.global_stop(current_time) + self.protections.stop_per_pair(pair, current_time, side) + self.protections.global_stop(current_time, side) def manage_open_orders(self, trade: LocalTrade, current_time, row: Tuple) -> bool: """ @@ -1030,7 +1031,7 @@ class Backtesting: and self.trade_slot_available(max_open_trades, open_trade_count_start) and current_time != end_date and trade_dir is not None - and not PairLocks.is_pair_locked(pair, row[DATE_IDX]) + and not PairLocks.is_pair_locked(pair, row[DATE_IDX], trade_dir) ): trade = self._enter_trade(pair, row, trade_dir) if trade: @@ -1068,7 +1069,8 @@ class Backtesting: LocalTrade.close_bt_trade(trade) trades.append(trade) self.wallets.update() - self.run_protections(enable_protections, pair, current_time) + self.run_protections( + enable_protections, pair, current_time, trade.trade_direction) # Move time one configured time_interval ahead. self.progress.increment() @@ -1092,7 +1094,7 @@ class Backtesting: timerange: TimeRange): self.progress.init_step(BacktestState.ANALYZE, 0) - logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) + logger.info(f"Running backtesting for Strategy {strat.get_strategy_name()}") backtest_start_time = datetime.now(timezone.utc) self._set_strategy(strat) diff --git a/freqtrade/optimize/hyperopt_loss_calmar.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py similarity index 96% rename from freqtrade/optimize/hyperopt_loss_calmar.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py index 846dae9ea..ea6c151e5 100644 --- a/freqtrade/optimize/hyperopt_loss_calmar.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py @@ -10,7 +10,7 @@ from typing import Any, Dict from pandas import DataFrame -from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss diff --git a/freqtrade/optimize/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py similarity index 95% rename from freqtrade/optimize/hyperopt_loss_max_drawdown.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py index ce955d928..a8af704cd 100644 --- a/freqtrade/optimize/hyperopt_loss_max_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown.py @@ -8,7 +8,7 @@ from datetime import datetime from pandas import DataFrame -from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss diff --git a/freqtrade/optimize/hyperopt_loss_onlyprofit.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_onlyprofit.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_onlyprofit.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_onlyprofit.py diff --git a/freqtrade/optimize/hyperopt_loss_profit_drawdown.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py similarity index 93% rename from freqtrade/optimize/hyperopt_loss_profit_drawdown.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py index 5bd12ff52..ed689edba 100644 --- a/freqtrade/optimize/hyperopt_loss_profit_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_profit_drawdown.py @@ -9,7 +9,7 @@ individual needs. """ from pandas import DataFrame -from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss diff --git a/freqtrade/optimize/hyperopt_loss_sharpe.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_sharpe.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe.py diff --git a/freqtrade/optimize/hyperopt_loss_sharpe_daily.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe_daily.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_sharpe_daily.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_sharpe_daily.py diff --git a/freqtrade/optimize/hyperopt_loss_short_trade_dur.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_short_trade_dur.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_short_trade_dur.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_short_trade_dur.py diff --git a/freqtrade/optimize/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_sortino.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino.py diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py similarity index 100% rename from freqtrade/optimize/hyperopt_loss_sortino_daily.py rename to freqtrade/optimize/hyperopt_loss/hyperopt_loss_sortino_daily.py diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 1d58dc339..9c1a276a9 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -9,8 +9,8 @@ from pandas import DataFrame, to_datetime from tabulate import tabulate from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT -from freqtrade.data.btanalysis import (calculate_cagr, calculate_csum, calculate_market_change, - calculate_max_drawdown) +from freqtrade.data.metrics import (calculate_cagr, calculate_csum, calculate_market_change, + calculate_max_drawdown) from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index f020f990c..03f3c3fb9 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -9,7 +9,7 @@ from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) -def get_table_names_for_table(inspector, tabletype): +def get_table_names_for_table(inspector, tabletype) -> List[str]: return [t for t in inspector.get_table_names() if t.startswith(tabletype)] @@ -21,7 +21,7 @@ def get_column_def(columns: List, column: str, default: str) -> str: return default if not has_column(columns, column) else column -def get_backup_name(tabs, backup_prefix: str): +def get_backup_name(tabs: List[str], backup_prefix: str): table_back_name = backup_prefix for i, table_back_name in enumerate(tabs): table_back_name = f'{backup_prefix}{i}' @@ -56,6 +56,16 @@ def set_sequence_ids(engine, order_id, trade_id): connection.execute(text(f"ALTER SEQUENCE trades_id_seq RESTART WITH {trade_id}")) +def drop_index_on_table(engine, inspector, table_bak_name): + with engine.begin() as connection: + # drop indexes on backup table in new session + for index in inspector.get_indexes(table_bak_name): + if engine.name == 'mysql': + connection.execute(text(f"drop index {index['name']} on {table_bak_name}")) + else: + connection.execute(text(f"drop index {index['name']}")) + + def migrate_trades_and_orders_table( decl_base, inspector, engine, trade_back_name: str, cols: List, @@ -116,13 +126,7 @@ def migrate_trades_and_orders_table( with engine.begin() as connection: connection.execute(text(f"alter table trades rename to {trade_back_name}")) - with engine.begin() as connection: - # drop indexes on backup table in new session - for index in inspector.get_indexes(trade_back_name): - if engine.name == 'mysql': - connection.execute(text(f"drop index {index['name']} on {trade_back_name}")) - else: - connection.execute(text(f"drop index {index['name']}")) + drop_index_on_table(engine, inspector, trade_back_name) order_id, trade_id = get_last_sequence_ids(engine, trade_back_name, order_back_name) @@ -205,6 +209,31 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List): """)) +def migrate_pairlocks_table( + decl_base, inspector, engine, + pairlock_back_name: str, cols: List): + + # Schema migration necessary + with engine.begin() as connection: + connection.execute(text(f"alter table pairlocks rename to {pairlock_back_name}")) + + drop_index_on_table(engine, inspector, pairlock_back_name) + + side = get_column_def(cols, 'side', "'*'") + + # let SQLAlchemy create the schema as required + decl_base.metadata.create_all(engine) + # Copy data back - following the correct schema + with engine.begin() as connection: + connection.execute(text(f"""insert into pairlocks + (id, pair, side, reason, lock_time, + lock_end_time, active) + select id, pair, {side} side, reason, lock_time, + lock_end_time, active + from {pairlock_back_name} + """)) + + def set_sqlite_to_wal(engine): if engine.name == 'sqlite' and str(engine.url) != 'sqlite://': # Set Mode to @@ -220,10 +249,13 @@ def check_migrate(engine, decl_base, previous_tables) -> None: cols_trades = inspector.get_columns('trades') cols_orders = inspector.get_columns('orders') + cols_pairlocks = inspector.get_columns('pairlocks') tabs = get_table_names_for_table(inspector, 'trades') table_back_name = get_backup_name(tabs, 'trades_bak') order_tabs = get_table_names_for_table(inspector, 'orders') order_table_bak_name = get_backup_name(order_tabs, 'orders_bak') + pairlock_tabs = get_table_names_for_table(inspector, 'pairlocks') + pairlock_table_bak_name = get_backup_name(pairlock_tabs, 'pairlocks_bak') # Check if migration necessary # Migrates both trades and orders table! @@ -236,6 +268,13 @@ def check_migrate(engine, decl_base, previous_tables) -> None: decl_base, inspector, engine, table_back_name, cols_trades, order_table_bak_name, cols_orders) + if not has_column(cols_pairlocks, 'side'): + logger.info(f"Running database migration for pairlocks - " + f"backup: {pairlock_table_bak_name}") + + migrate_pairlocks_table( + decl_base, inspector, engine, pairlock_table_bak_name, cols_pairlocks + ) if 'orders' not in previous_tables and 'trades' in previous_tables: raise OperationalException( "Your database seems to be very old. " diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 2cacc06e2..299032bb4 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -7,13 +7,13 @@ from decimal import Decimal from typing import Any, Dict, List, Optional from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String, - create_engine, desc, func, inspect) + create_engine, desc, func, inspect, or_) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session, sessionmaker from sqlalchemy.pool import StaticPool from sqlalchemy.sql.schema import UniqueConstraint -from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES +from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES, LongShort from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.leverage import interest @@ -393,7 +393,7 @@ class LocalTrade(): return "sell" @property - def trade_direction(self) -> str: + def trade_direction(self) -> LongShort: if self.is_short: return "short" else: @@ -1426,6 +1426,8 @@ class PairLock(_DECL_BASE): id = Column(Integer, primary_key=True) pair = Column(String(25), nullable=False, index=True) + # lock direction - long, short or * (for both) + side = Column(String(25), nullable=False, default="*") reason = Column(String(255), nullable=True) # Time the pair was locked (start time) lock_time = Column(DateTime, nullable=False) @@ -1437,11 +1439,12 @@ class PairLock(_DECL_BASE): def __repr__(self): lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT) lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT) - return (f'PairLock(id={self.id}, pair={self.pair}, lock_time={lock_time}, ' - f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})') + return ( + f'PairLock(id={self.id}, pair={self.pair}, side={self.side}, lock_time={lock_time}, ' + f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})') @staticmethod - def query_pair_locks(pair: Optional[str], now: datetime) -> Query: + def query_pair_locks(pair: Optional[str], now: datetime, side: str = '*') -> Query: """ Get all currently active locks for this pair :param pair: Pair to check for. Returns all current locks if pair is empty @@ -1452,6 +1455,11 @@ class PairLock(_DECL_BASE): PairLock.active.is_(True), ] if pair: filters.append(PairLock.pair == pair) + if side != '*': + filters.append(or_(PairLock.side == side, PairLock.side == '*')) + else: + filters.append(PairLock.side == '*') + return PairLock.query.filter( *filters ) @@ -1466,5 +1474,6 @@ class PairLock(_DECL_BASE): 'lock_end_timestamp': int(self.lock_end_time.replace(tzinfo=timezone.utc ).timestamp() * 1000), 'reason': self.reason, + 'side': self.side, 'active': self.active, } diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index afbd9781b..ec57e91fc 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -31,7 +31,7 @@ class PairLocks(): @staticmethod def lock_pair(pair: str, until: datetime, reason: str = None, *, - now: datetime = None) -> PairLock: + now: datetime = None, side: str = '*') -> PairLock: """ Create PairLock from now to "until". Uses database by default, unless PairLocks.use_db is set to False, @@ -40,12 +40,14 @@ class PairLocks(): :param until: End time of the lock. Will be rounded up to the next candle. :param reason: Reason string that will be shown as reason for the lock :param now: Current timestamp. Used to determine lock start time. + :param side: Side to lock pair, can be 'long', 'short' or '*' """ lock = PairLock( pair=pair, lock_time=now or datetime.now(timezone.utc), lock_end_time=timeframe_to_next_date(PairLocks.timeframe, until), reason=reason, + side=side, active=True ) if PairLocks.use_db: @@ -56,7 +58,8 @@ class PairLocks(): return lock @staticmethod - def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]: + def get_pair_locks( + pair: Optional[str], now: Optional[datetime] = None, side: str = '*') -> List[PairLock]: """ Get all currently active locks for this pair :param pair: Pair to check for. Returns all current locks if pair is empty @@ -67,26 +70,28 @@ class PairLocks(): now = datetime.now(timezone.utc) if PairLocks.use_db: - return PairLock.query_pair_locks(pair, now).all() + return PairLock.query_pair_locks(pair, now, side).all() else: locks = [lock for lock in PairLocks.locks if ( lock.lock_end_time >= now and lock.active is True and (pair is None or lock.pair == pair) + and (lock.side == '*' or lock.side == side) )] return locks @staticmethod - def get_pair_longest_lock(pair: str, now: Optional[datetime] = None) -> Optional[PairLock]: + def get_pair_longest_lock( + pair: str, now: Optional[datetime] = None, side: str = '*') -> Optional[PairLock]: """ Get the lock that expires the latest for the pair given. """ - locks = PairLocks.get_pair_locks(pair, now) + locks = PairLocks.get_pair_locks(pair, now, side=side) locks = sorted(locks, key=lambda l: l.lock_end_time, reverse=True) return locks[0] if locks else None @staticmethod - def unlock_pair(pair: str, now: Optional[datetime] = None) -> None: + def unlock_pair(pair: str, now: Optional[datetime] = None, side: str = '*') -> None: """ Release all locks for this pair. :param pair: Pair to unlock @@ -97,7 +102,7 @@ class PairLocks(): now = datetime.now(timezone.utc) logger.info(f"Releasing all locks for {pair}.") - locks = PairLocks.get_pair_locks(pair, now) + locks = PairLocks.get_pair_locks(pair, now, side=side) for lock in locks: lock.active = False if PairLocks.use_db: @@ -134,7 +139,7 @@ class PairLocks(): lock.active = False @staticmethod - def is_global_lock(now: Optional[datetime] = None) -> bool: + def is_global_lock(now: Optional[datetime] = None, side: str = '*') -> bool: """ :param now: Datetime object (generated via datetime.now(timezone.utc)). defaults to datetime.now(timezone.utc) @@ -142,10 +147,10 @@ class PairLocks(): if not now: now = datetime.now(timezone.utc) - return len(PairLocks.get_pair_locks('*', now)) > 0 + return len(PairLocks.get_pair_locks('*', now, side)) > 0 @staticmethod - def is_pair_locked(pair: str, now: Optional[datetime] = None) -> bool: + def is_pair_locked(pair: str, now: Optional[datetime] = None, side: str = '*') -> bool: """ :param pair: Pair to check for :param now: Datetime object (generated via datetime.now(timezone.utc)). @@ -154,7 +159,10 @@ class PairLocks(): if not now: now = datetime.now(timezone.utc) - return len(PairLocks.get_pair_locks(pair, now)) > 0 or PairLocks.is_global_lock(now) + return ( + len(PairLocks.get_pair_locks(pair, now, side)) > 0 + or PairLocks.is_global_lock(now, side) + ) @staticmethod def get_all_locks() -> List[PairLock]: diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 5337016f3..773577d7b 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -5,12 +5,13 @@ from typing import Any, Dict, List, Optional import pandas as pd from freqtrade.configuration import TimeRange -from freqtrade.data.btanalysis import (analyze_trade_parallelism, calculate_max_drawdown, - calculate_underwater, combine_dataframes_with_mean, - create_cum_profit, extract_trades_of_period, load_trades) +from freqtrade.data.btanalysis import (analyze_trade_parallelism, extract_trades_of_period, + load_trades) from freqtrade.data.converter import trim_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange, load_data +from freqtrade.data.metrics import (calculate_max_drawdown, calculate_underwater, + combine_dataframes_with_mean, create_cum_profit) from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index 2510d6fee..d33294fa7 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -5,6 +5,7 @@ import logging from datetime import datetime, timezone from typing import Dict, List, Optional +from freqtrade.constants import LongShort from freqtrade.persistence import PairLocks from freqtrade.persistence.models import PairLock from freqtrade.plugins.protections import IProtection @@ -44,28 +45,31 @@ class ProtectionManager(): """ return [{p.name: p.short_desc()} for p in self._protection_handlers] - def global_stop(self, now: Optional[datetime] = None) -> Optional[PairLock]: + def global_stop(self, now: Optional[datetime] = None, + side: LongShort = 'long') -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) result = None for protection_handler in self._protection_handlers: if protection_handler.has_global_stop: - lock, until, reason = protection_handler.global_stop(now) - - # Early stopping - first positive result blocks further trades - if lock and until: - if not PairLocks.is_global_lock(until): - result = PairLocks.lock_pair('*', until, reason, now=now) + lock = protection_handler.global_stop(date_now=now, side=side) + if lock and lock.until: + if not PairLocks.is_global_lock(lock.until, side=lock.lock_side): + result = PairLocks.lock_pair( + '*', lock.until, lock.reason, now=now, side=lock.lock_side) return result - def stop_per_pair(self, pair, now: Optional[datetime] = None) -> Optional[PairLock]: + def stop_per_pair(self, pair, now: Optional[datetime] = None, + side: LongShort = 'long') -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) result = None for protection_handler in self._protection_handlers: if protection_handler.has_local_stop: - lock, until, reason = protection_handler.stop_per_pair(pair, now) - if lock and until: - if not PairLocks.is_pair_locked(pair, until): - result = PairLocks.lock_pair(pair, until, reason, now=now) + lock = protection_handler.stop_per_pair( + pair=pair, date_now=now, side=side) + if lock and lock.until: + if not PairLocks.is_pair_locked(pair, lock.until, lock.lock_side): + result = PairLocks.lock_pair( + pair, lock.until, lock.reason, now=now, side=lock.lock_side) return result diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index a2d8eca34..426b8f1b6 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -1,7 +1,9 @@ import logging from datetime import datetime, timedelta +from typing import Optional +from freqtrade.constants import LongShort from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -26,7 +28,7 @@ class CooldownPeriod(IProtection): """ return (f"{self.name} - Cooldown period of {self.stop_duration_str}.") - def _cooldown_period(self, pair: str, date_now: datetime, ) -> ProtectionReturn: + def _cooldown_period(self, pair: str, date_now: datetime) -> Optional[ProtectionReturn]: """ Get last trade for this pair """ @@ -45,11 +47,15 @@ class CooldownPeriod(IProtection): self.log_once(f"Cooldown for {pair} for {self.stop_duration_str}.", logger.info) until = self.calculate_lock_end([trade], self._stop_duration) - return True, until, self._reason() + return ProtectionReturn( + lock=True, + until=until, + reason=self._reason(), + ) - return False, None, None + return None - def global_stop(self, date_now: datetime) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -57,9 +63,10 @@ class CooldownPeriod(IProtection): If true, all pairs will be locked with until """ # Not implemented for cooldown period. - return False, None, None + return None - def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + def stop_per_pair( + self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index e0a89e334..890988226 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -1,9 +1,11 @@ import logging from abc import ABC, abstractmethod +from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional +from freqtrade.constants import LongShort from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import plural from freqtrade.mixins import LoggingMixin @@ -12,7 +14,13 @@ from freqtrade.persistence import LocalTrade logger = logging.getLogger(__name__) -ProtectionReturn = Tuple[bool, Optional[datetime], Optional[str]] + +@dataclass +class ProtectionReturn: + lock: bool + until: datetime + reason: Optional[str] + lock_side: str = '*' class IProtection(LoggingMixin, ABC): @@ -80,14 +88,15 @@ class IProtection(LoggingMixin, ABC): """ @abstractmethod - def global_stop(self, date_now: datetime) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". """ @abstractmethod - def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + def stop_per_pair( + self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index 7822ce73c..7d5d6054d 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -1,8 +1,9 @@ import logging from datetime import datetime, timedelta -from typing import Any, Dict +from typing import Any, Dict, Optional +from freqtrade.constants import LongShort from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -35,7 +36,7 @@ class LowProfitPairs(IProtection): return (f'{profit} < {self._required_profit} in {self.lookback_period_str}, ' f'locking for {self.stop_duration_str}.') - def _low_profit(self, date_now: datetime, pair: str) -> ProtectionReturn: + def _low_profit(self, date_now: datetime, pair: str) -> Optional[ProtectionReturn]: """ Evaluate recent trades for pair """ @@ -51,7 +52,7 @@ class LowProfitPairs(IProtection): # trades = Trade.get_trades(filters).all() if len(trades) < self._trade_limit: # Not enough trades in the relevant period - return False, None, None + return None profit = sum(trade.close_profit for trade in trades if trade.close_profit) if profit < self._required_profit: @@ -60,20 +61,25 @@ class LowProfitPairs(IProtection): f"within {self._lookback_period} minutes.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) - return True, until, self._reason(profit) + return ProtectionReturn( + lock=True, + until=until, + reason=self._reason(profit), + ) - return False, None, None + return None - def global_stop(self, date_now: datetime) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". :return: Tuple of [bool, until, reason]. If true, all pairs will be locked with until """ - return False, None, None + return None - def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + def stop_per_pair( + self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index b6ef92bd5..e0b016cb8 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -1,11 +1,12 @@ import logging from datetime import datetime, timedelta -from typing import Any, Dict +from typing import Any, Dict, Optional import pandas as pd -from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.constants import LongShort +from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -39,7 +40,7 @@ class MaxDrawdown(IProtection): return (f'{drawdown} passed {self._max_allowed_drawdown} in {self.lookback_period_str}, ' f'locking for {self.stop_duration_str}.') - def _max_drawdown(self, date_now: datetime) -> ProtectionReturn: + def _max_drawdown(self, date_now: datetime) -> Optional[ProtectionReturn]: """ Evaluate recent trades for drawdown ... """ @@ -51,14 +52,14 @@ class MaxDrawdown(IProtection): if len(trades) < self._trade_limit: # Not enough trades in the relevant period - return False, None, None + return None # Drawdown is always positive try: # TODO: This should use absolute profit calculation, considering account balance. drawdown, _, _, _, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit') except ValueError: - return False, None, None + return None if drawdown > self._max_allowed_drawdown: self.log_once( @@ -66,11 +67,15 @@ class MaxDrawdown(IProtection): f" within {self.lookback_period_str}.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) - return True, until, self._reason(drawdown) + return ProtectionReturn( + lock=True, + until=until, + reason=self._reason(drawdown), + ) - return False, None, None + return None - def global_stop(self, date_now: datetime) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -79,11 +84,12 @@ class MaxDrawdown(IProtection): """ return self._max_drawdown(date_now) - def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + def stop_per_pair( + self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". :return: Tuple of [bool, until, reason]. If true, this pair will be locked with until """ - return False, None, None + return None diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 8d7fb2a0e..f9fe039d6 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -1,8 +1,9 @@ import logging from datetime import datetime, timedelta -from typing import Any, Dict +from typing import Any, Dict, Optional +from freqtrade.constants import LongShort from freqtrade.enums import ExitType from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -21,6 +22,7 @@ class StoplossGuard(IProtection): self._trade_limit = protection_config.get('trade_limit', 10) self._disable_global_stop = protection_config.get('only_per_pair', False) + self._only_per_side = protection_config.get('only_per_side', False) def short_desc(self) -> str: """ @@ -36,7 +38,8 @@ class StoplossGuard(IProtection): return (f'{self._trade_limit} stoplosses in {self._lookback_period} min, ' f'locking for {self._stop_duration} min.') - def _stoploss_guard(self, date_now: datetime, pair: str = None) -> ProtectionReturn: + def _stoploss_guard( + self, date_now: datetime, pair: Optional[str], side: str) -> Optional[ProtectionReturn]: """ Evaluate recent trades """ @@ -48,15 +51,24 @@ class StoplossGuard(IProtection): ExitType.STOPLOSS_ON_EXCHANGE.value) and trade.close_profit and trade.close_profit < 0)] + if self._only_per_side: + # Long or short trades only + trades = [trade for trade in trades if trade.trade_direction == side] + if len(trades) < self._trade_limit: - return False, None, None + return None self.log_once(f"Trading stopped due to {self._trade_limit} " f"stoplosses within {self._lookback_period} minutes.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) - return True, until, self._reason() + return ProtectionReturn( + lock=True, + until=until, + reason=self._reason(), + lock_side=(side if self._only_per_side else '*') + ) - def global_stop(self, date_now: datetime) -> ProtectionReturn: + def global_stop(self, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". @@ -64,14 +76,15 @@ class StoplossGuard(IProtection): If true, all pairs will be locked with until """ if self._disable_global_stop: - return False, None, None - return self._stoploss_guard(date_now, None) + return None + return self._stoploss_guard(date_now, None, side) - def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + def stop_per_pair( + self, pair: str, date_now: datetime, side: LongShort) -> Optional[ProtectionReturn]: """ Stops trading (position entering) for this pair This must evaluate to true for the whole period of the "cooldown period". :return: Tuple of [bool, until, reason]. If true, this pair will be locked with until """ - return self._stoploss_guard(date_now, pair) + return self._stoploss_guard(date_now, pair, side) diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index e3c234f60..bcfe5e1d8 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -23,7 +23,7 @@ class HyperOptLossResolver(IResolver): object_type = IHyperOptLoss object_type_str = "HyperoptLoss" user_subdir = USERPATH_HYPEROPTS - initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve() + initial_search_path = Path(__file__).parent.parent.joinpath('optimize/hyperopt_loss').resolve() @staticmethod def load_hyperoptloss(config: Dict) -> IHyperOptLoss: diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index a902ea984..41712632b 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -84,6 +84,7 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac lastconfig['enable_protections'] = btconfig.get('enable_protections') lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') + ApiServer._bt.strategylist = [strat] ApiServer._bt.results = {} ApiServer._bt.load_prior_backtest() diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index a9135cce2..d78ea8b78 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -291,6 +291,7 @@ class LockModel(BaseModel): lock_time: str lock_timestamp: int pair: str + side: str reason: str diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 1ea5f2d5d..367097d71 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -573,7 +573,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ return self.__class__.__name__ - def lock_pair(self, pair: str, until: datetime, reason: str = None) -> None: + def lock_pair(self, pair: str, until: datetime, reason: str = None, side: str = '*') -> None: """ Locks pair until a given timestamp happens. Locked pairs are not analyzed, and are prevented from opening new trades. @@ -583,8 +583,9 @@ class IStrategy(ABC, HyperStrategyMixin): :param until: datetime in UTC until the pair should be blocked from opening new trades. Needs to be timezone aware `datetime.now(timezone.utc)` :param reason: Optional string explaining why the pair was locked. + :param side: Side to check, can be long, short or '*' """ - PairLocks.lock_pair(pair, until, reason) + PairLocks.lock_pair(pair, until, reason, side=side) def unlock_pair(self, pair: str) -> None: """ @@ -604,7 +605,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ PairLocks.unlock_reason(reason, datetime.now(timezone.utc)) - def is_pair_locked(self, pair: str, candle_date: datetime = None) -> bool: + def is_pair_locked(self, pair: str, *, candle_date: datetime = None, side: str = '*') -> bool: """ Checks if a pair is currently locked The 2nd, optional parameter ensures that locks are applied until the new candle arrives, @@ -612,15 +613,16 @@ class IStrategy(ABC, HyperStrategyMixin): of 2 seconds for an entry order to happen on an old signal. :param pair: "Pair to check" :param candle_date: Date of the last candle. Optional, defaults to current date + :param side: Side to check, can be long, short or '*' :returns: locking state of the pair in question. """ if not candle_date: # Simple call ... - return PairLocks.is_pair_locked(pair) + return PairLocks.is_pair_locked(pair, side=side) else: lock_time = timeframe_to_next_date(self.timeframe, candle_date) - return PairLocks.is_pair_locked(pair, lock_time) + return PairLocks.is_pair_locked(pair, lock_time, side=side) def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index eaf703b2d..f9f49e280 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -8,14 +8,14 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import LAST_BT_RESULT_FN -from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, calculate_cagr, - calculate_csum, calculate_market_change, - calculate_max_drawdown, calculate_underwater, - combine_dataframes_with_mean, create_cum_profit, +from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, extract_trades_of_period, get_latest_backtest_filename, get_latest_hyperopt_file, load_backtest_data, load_backtest_metadata, load_trades, load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history +from freqtrade.data.metrics import (calculate_cagr, calculate_csum, calculate_market_change, + calculate_max_drawdown, calculate_underwater, + combine_dataframes_with_mean, create_cum_profit) from freqtrade.exceptions import OperationalException from tests.conftest import CURRENT_TEST_STRATEGY, create_mock_trades from tests.conftest_trades import MOCK_TRADE_COUNT diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 0585fa0d4..82d4a841c 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -149,8 +149,8 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC', candle_type=candle_type) assert file.is_file() assert log_has_re( - r'Download history data for pair: "MEME/BTC" \(0/1\), timeframe: 1m, ' - r'candle type: spot and store in .*', caplog + r'\(0/1\) - Download history data for "MEME/BTC", 1m, ' + r'spot and store in .*', caplog ) @@ -223,42 +223,65 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None: # timeframe starts earlier than the cached data # should fully update data timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) - data, start_ts = _load_cached_data_for_updating( + data, start_ts, end_ts = _load_cached_data_for_updating( 'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT) assert data.empty assert start_ts == test_data[0][0] - 1000 + assert end_ts is None + + # timeframe starts earlier than the cached data - prepending + + timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) + data, start_ts, end_ts = _load_cached_data_for_updating( + 'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT, True) + assert_frame_equal(data, test_data_df.iloc[:-1]) + assert start_ts == test_data[0][0] - 1000 + assert end_ts == test_data[0][0] # timeframe starts in the center of the cached data # should return the cached data w/o the last item timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) - data, start_ts = _load_cached_data_for_updating( + data, start_ts, end_ts = _load_cached_data_for_updating( 'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT) assert_frame_equal(data, test_data_df.iloc[:-1]) assert test_data[-2][0] <= start_ts < test_data[-1][0] + assert end_ts is None # timeframe starts after the cached data # should return the cached data w/o the last item timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 100, 0) - data, start_ts = _load_cached_data_for_updating( + data, start_ts, end_ts = _load_cached_data_for_updating( 'UNITTEST/BTC', '1m', timerange, data_handler, CandleType.SPOT) assert_frame_equal(data, test_data_df.iloc[:-1]) assert test_data[-2][0] <= start_ts < test_data[-1][0] + assert end_ts is None # no datafile exist # should return timestamp start time timerange = TimeRange('date', None, now_ts - 10000, 0) - data, start_ts = _load_cached_data_for_updating( + data, start_ts, end_ts = _load_cached_data_for_updating( 'NONEXIST/BTC', '1m', timerange, data_handler, CandleType.SPOT) assert data.empty assert start_ts == (now_ts - 10000) * 1000 + assert end_ts is None + + # no datafile exist + # should return timestamp start and end time time + timerange = TimeRange('date', 'date', now_ts - 1000000, now_ts - 100000) + data, start_ts, end_ts = _load_cached_data_for_updating( + 'NONEXIST/BTC', '1m', timerange, data_handler, CandleType.SPOT) + assert data.empty + assert start_ts == (now_ts - 1000000) * 1000 + assert end_ts == (now_ts - 100000) * 1000 # no datafile exist, no timeframe is set # should return an empty array and None - data, start_ts = _load_cached_data_for_updating( + data, start_ts, end_ts = _load_cached_data_for_updating( 'NONEXIST/BTC', '1m', None, data_handler, CandleType.SPOT) assert data.empty assert start_ts is None + assert end_ts is None @pytest.mark.parametrize('candle_type,subdir,file_tail', [ diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 689ffa4ce..31311cc38 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1983,6 +1983,20 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ assert exchange._api_async.fetch_ohlcv.call_count > 200 assert res[0] == ohlcv[0] + exchange._api_async.fetch_ohlcv.reset_mock() + end_ts = 1_500_500_000_000 + start_ts = 1_500_000_000_000 + respair, restf, _, res = await exchange._async_get_historic_ohlcv( + pair, "5m", since_ms=start_ts, candle_type=candle_type, is_new_pair=False, + until_ms=end_ts + ) + # Required candles + candles = (end_ts - start_ts) / 300_000 + exp = candles // exchange.ohlcv_candle_limit('5m') + 1 + + # Depending on the exchange, this should be called between 1 and 6 times. + assert exchange._api_async.fetch_ohlcv.call_count == exp + @pytest.mark.parametrize('candle_type', [CandleType.FUTURES, CandleType.MARK, CandleType.SPOT]) def test_refresh_latest_ohlcv(mocker, default_conf, caplog, candle_type) -> None: diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py index e3f6daf6c..4ec80ef49 100644 --- a/tests/optimize/test_hyperoptloss.py +++ b/tests/optimize/test_hyperoptloss.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock import pytest from freqtrade.exceptions import OperationalException -from freqtrade.optimize.hyperopt_loss_short_trade_dur import ShortTradeDurHyperOptLoss +from freqtrade.optimize.hyperopt_loss.hyperopt_loss_short_trade_dur import ShortTradeDurHyperOptLoss from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver diff --git a/tests/plugins/test_pairlocks.py b/tests/plugins/test_pairlocks.py index f9e5583ed..0ba9bb746 100644 --- a/tests/plugins/test_pairlocks.py +++ b/tests/plugins/test_pairlocks.py @@ -21,8 +21,22 @@ def test_PairLocks(use_db): pair = 'ETH/BTC' assert not PairLocks.is_pair_locked(pair) PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) - # ETH/BTC locked for 4 minutes + # ETH/BTC locked for 4 minutes (on both sides) assert PairLocks.is_pair_locked(pair) + assert PairLocks.is_pair_locked(pair, side='long') + assert PairLocks.is_pair_locked(pair, side='short') + + pair = 'BNB/BTC' + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime, side='long') + assert not PairLocks.is_pair_locked(pair) + assert PairLocks.is_pair_locked(pair, side='long') + assert not PairLocks.is_pair_locked(pair, side='short') + + pair = 'BNB/USDT' + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime, side='short') + assert not PairLocks.is_pair_locked(pair) + assert not PairLocks.is_pair_locked(pair, side='long') + assert PairLocks.is_pair_locked(pair, side='short') # XRP/BTC should not be locked now pair = 'XRP/BTC' diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 6b69f5481..b2dc99610 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -11,9 +11,10 @@ from tests.conftest import get_patched_freqtradebot, log_has_re def generate_mock_trade(pair: str, fee: float, is_open: bool, - sell_reason: str = ExitType.EXIT_SIGNAL, + exit_reason: str = ExitType.EXIT_SIGNAL, min_ago_open: int = None, min_ago_close: int = None, - profit_rate: float = 0.9 + profit_rate: float = 0.9, + is_short: bool = False, ): open_rate = random.random() @@ -28,11 +29,12 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool, is_open=is_open, amount=0.01 / open_rate, exchange='binance', + is_short=is_short, ) trade.recalc_open_trade_value() if not is_open: - trade.close(open_rate * profit_rate) - trade.exit_reason = sell_reason + trade.close(open_rate * (2 - profit_rate if is_short else profit_rate)) + trade.exit_reason = exit_reason return trade @@ -45,9 +47,9 @@ def test_protectionmanager(mocker, default_conf): for handler in freqtrade.protections._protection_handlers: assert handler.name in constants.AVAILABLE_PROTECTIONS if not handler.has_global_stop: - assert handler.global_stop(datetime.utcnow()) == (False, None, None) + assert handler.global_stop(datetime.utcnow(), '*') is None if not handler.has_local_stop: - assert handler.stop_per_pair('XRP/BTC', datetime.utcnow()) == (False, None, None) + assert handler.stop_per_pair('XRP/BTC', datetime.utcnow(), '*') is None @pytest.mark.parametrize('timeframe,expected,protconf', [ @@ -68,7 +70,7 @@ def test_protectionmanager(mocker, default_conf): ('1h', [60, 540], [{"method": "StoplossGuard", "lookback_period_candles": 1, "stop_duration_candles": 9}]), ]) -def test_protections_init(mocker, default_conf, timeframe, expected, protconf): +def test_protections_init(default_conf, timeframe, expected, protconf): default_conf['timeframe'] = timeframe man = ProtectionManager(default_conf, protconf) assert len(man._protection_handlers) == len(protconf) @@ -76,8 +78,10 @@ def test_protections_init(mocker, default_conf, timeframe, expected, protconf): assert man._protection_handlers[0]._stop_duration == expected[1] +@pytest.mark.parametrize('is_short', [False, True]) @pytest.mark.usefixtures("init_persistence") -def test_stoploss_guard(mocker, default_conf, fee, caplog): +def test_stoploss_guard(mocker, default_conf, fee, caplog, is_short): + # Active for both sides (long and short) default_conf['protections'] = [{ "method": "StoplossGuard", "lookback_period": 60, @@ -91,8 +95,8 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): caplog.clear() Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, - min_ago_open=200, min_ago_close=30, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=30, is_short=is_short, )) assert not freqtrade.protections.global_stop() @@ -100,13 +104,13 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): caplog.clear() # This trade does not count, as it's closed too long ago Trade.query.session.add(generate_mock_trade( - 'BCH/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, - min_ago_open=250, min_ago_close=100, + 'BCH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=250, min_ago_close=100, is_short=is_short, )) Trade.query.session.add(generate_mock_trade( - 'ETH/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, - min_ago_open=240, min_ago_close=30, + 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=240, min_ago_close=30, is_short=is_short, )) # 3 Trades closed - but the 2nd has been closed too long ago. assert not freqtrade.protections.global_stop() @@ -114,8 +118,8 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): caplog.clear() Trade.query.session.add(generate_mock_trade( - 'LTC/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, - min_ago_open=180, min_ago_close=30, + 'LTC/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=180, min_ago_close=30, is_short=is_short, )) assert freqtrade.protections.global_stop() @@ -130,15 +134,19 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): @pytest.mark.parametrize('only_per_pair', [False, True]) +@pytest.mark.parametrize('only_per_side', [False, True]) @pytest.mark.usefixtures("init_persistence") -def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair): +def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair, only_per_side): default_conf['protections'] = [{ "method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2, "stop_duration": 60, - "only_per_pair": only_per_pair + "only_per_pair": only_per_pair, + "only_per_side": only_per_side, }] + check_side = 'long' if only_per_side else '*' + is_short = False freqtrade = get_patched_freqtradebot(mocker, default_conf) message = r"Trading stopped due to .*" pair = 'XRP/BTC' @@ -148,8 +156,8 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair caplog.clear() Trade.query.session.add(generate_mock_trade( - pair, fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, - min_ago_open=200, min_ago_close=30, profit_rate=0.9, + pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=30, profit_rate=0.9, is_short=is_short )) assert not freqtrade.protections.stop_per_pair(pair) @@ -158,13 +166,13 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair caplog.clear() # This trade does not count, as it's closed too long ago Trade.query.session.add(generate_mock_trade( - pair, fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, - min_ago_open=250, min_ago_close=100, profit_rate=0.9, + pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=250, min_ago_close=100, profit_rate=0.9, is_short=is_short )) # Trade does not count for per pair stop as it's the wrong pair. Trade.query.session.add(generate_mock_trade( - 'ETH/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, - min_ago_open=240, min_ago_close=30, profit_rate=0.9, + 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=240, min_ago_close=30, profit_rate=0.9, is_short=is_short )) # 3 Trades closed - but the 2nd has been closed too long ago. assert not freqtrade.protections.stop_per_pair(pair) @@ -176,16 +184,34 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair caplog.clear() + # Trade does not count potentially, as it's in the wrong direction + Trade.query.session.add(generate_mock_trade( + pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=150, min_ago_close=25, profit_rate=0.9, is_short=not is_short + )) + freqtrade.protections.stop_per_pair(pair) + assert freqtrade.protections.global_stop() != only_per_pair + assert PairLocks.is_pair_locked(pair, side=check_side) != (only_per_side and only_per_pair) + assert PairLocks.is_global_lock(side=check_side) != only_per_pair + if only_per_side: + assert not PairLocks.is_pair_locked(pair, side='*') + assert not PairLocks.is_global_lock(side='*') + + caplog.clear() + # 2nd Trade that counts with correct pair Trade.query.session.add(generate_mock_trade( - pair, fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, - min_ago_open=180, min_ago_close=30, profit_rate=0.9, + pair, fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, + min_ago_open=180, min_ago_close=30, profit_rate=0.9, is_short=is_short )) freqtrade.protections.stop_per_pair(pair) assert freqtrade.protections.global_stop() != only_per_pair - assert PairLocks.is_pair_locked(pair) - assert PairLocks.is_global_lock() != only_per_pair + assert PairLocks.is_pair_locked(pair, side=check_side) + assert PairLocks.is_global_lock(side=check_side) != only_per_pair + if only_per_side: + assert not PairLocks.is_pair_locked(pair, side='*') + assert not PairLocks.is_global_lock(side='*') @pytest.mark.usefixtures("init_persistence") @@ -203,7 +229,7 @@ def test_CooldownPeriod(mocker, default_conf, fee, caplog): caplog.clear() Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=200, min_ago_close=30, )) @@ -213,7 +239,7 @@ def test_CooldownPeriod(mocker, default_conf, fee, caplog): assert not PairLocks.is_global_lock() Trade.query.session.add(generate_mock_trade( - 'ETH/BTC', fee.return_value, False, sell_reason=ExitType.ROI.value, + 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value, min_ago_open=205, min_ago_close=35, )) @@ -242,7 +268,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): caplog.clear() Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=800, min_ago_close=450, profit_rate=0.9, )) @@ -253,7 +279,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): assert not PairLocks.is_global_lock() Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=200, min_ago_close=120, profit_rate=0.9, )) @@ -265,14 +291,14 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): # Add positive trade Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.ROI.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value, min_ago_open=20, min_ago_close=10, profit_rate=1.15, )) assert not freqtrade.protections.stop_per_pair('XRP/BTC') assert not PairLocks.is_pair_locked('XRP/BTC') Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=110, min_ago_close=20, profit_rate=0.8, )) @@ -300,15 +326,15 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): caplog.clear() Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, )) Trade.query.session.add(generate_mock_trade( - 'ETH/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'ETH/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, )) Trade.query.session.add(generate_mock_trade( - 'NEO/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'NEO/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, )) # No losing trade yet ... so max_drawdown will raise exception @@ -316,7 +342,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): assert not freqtrade.protections.stop_per_pair('XRP/BTC') Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=500, min_ago_close=400, profit_rate=0.9, )) # Not locked with one trade @@ -326,7 +352,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): assert not PairLocks.is_global_lock() Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.STOP_LOSS.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.STOP_LOSS.value, min_ago_open=1200, min_ago_close=1100, profit_rate=0.5, )) @@ -339,7 +365,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): # Winning trade ... (should not lock, does not change drawdown!) Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.ROI.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value, min_ago_open=320, min_ago_close=410, profit_rate=1.5, )) assert not freqtrade.protections.global_stop() @@ -349,7 +375,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): # Add additional negative trade, causing a loss of > 15% Trade.query.session.add(generate_mock_trade( - 'XRP/BTC', fee.return_value, False, sell_reason=ExitType.ROI.value, + 'XRP/BTC', fee.return_value, False, exit_reason=ExitType.ROI.value, min_ago_open=20, min_ago_close=10, profit_rate=0.8, )) assert not freqtrade.protections.stop_per_pair('XRP/BTC') diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 43f783a53..ac2f1c3ec 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1483,7 +1483,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): assert not result['running'] assert result['status_msg'] == 'Backtest reset' ftbot.config['export'] = 'trades' - ftbot.config['backtest_cache'] = 'none' + ftbot.config['backtest_cache'] = 'day' ftbot.config['user_data_dir'] = Path(tmpdir) ftbot.config['exportfilename'] = Path(tmpdir) / "backtest_results" ftbot.config['exportfilename'].mkdir() @@ -1556,19 +1556,19 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): ApiServer._bgtask_running = False - mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', - side_effect=DependencyException()) - rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) - assert log_has("Backtesting caused an error: ", caplog) - - ftbot.config['backtest_cache'] = 'day' - # Rerun backtest (should get previous result) rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) assert_response(rc) result = rc.json() assert log_has_re('Reusing result of previous backtest.*', caplog) + data['stake_amount'] = 101 + + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', + side_effect=DependencyException()) + rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) + assert log_has("Backtesting caused an error: ", caplog) + # Delete backtesting to avoid leakage since the backtest-object may stick around. rc = client_delete(client, f"{BASE_URI}/backtest") assert_response(rc) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index ea81fe968..6e57a3182 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -666,23 +666,23 @@ def test_is_pair_locked(default_conf): assert not strategy.is_pair_locked(pair) # latest candle is from 14:20, lock goes to 14:30 - assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-10)) - assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-50)) + assert strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-10)) + assert strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-50)) # latest candle is from 14:25 (lock should be lifted) # Since this is the "new candle" available at 14:30 - assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-4)) + assert not strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-4)) # Should not be locked after time expired - assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=10)) + assert not strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=10)) # Change timeframe to 15m strategy.timeframe = '15m' # Candle from 14:14 - lock goes until 14:30 - assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-16)) - assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-15, seconds=-2)) + assert strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-16)) + assert strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-15, seconds=-2)) # Candle from 14:15 - lock goes until 14:30 - assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-15)) + assert not strategy.is_pair_locked(pair, candle_date=lock_time + timedelta(minutes=-15)) def test_is_informative_pairs_callback(default_conf): diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 2bef11de2..e3785e67e 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -21,6 +21,7 @@ from freqtrade.exceptions import (DependencyException, ExchangeError, Insufficie from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.persistence.models import PairLock +from freqtrade.plugins.protections.iprotection import ProtectionReturn from freqtrade.worker import Worker from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal, @@ -420,7 +421,7 @@ def test_enter_positions_global_pairlock(default_conf_usdt, ticker_usdt, limit_b assert not log_has_re(message, caplog) caplog.clear() - PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because') + PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because', side='*') n = freqtrade.enter_positions() assert n == 0 assert log_has_re(message, caplog) @@ -441,9 +442,9 @@ def test_handle_protections(mocker, default_conf_usdt, fee, is_short): freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) freqtrade.protections._protection_handlers[1].global_stop = MagicMock( - return_value=(True, arrow.utcnow().shift(hours=1).datetime, "asdf")) + return_value=ProtectionReturn(True, arrow.utcnow().shift(hours=1).datetime, "asdf")) create_mock_trades(fee, is_short) - freqtrade.handle_protections('ETC/BTC') + freqtrade.handle_protections('ETC/BTC', '*') send_msg_mock = freqtrade.rpc.send_msg assert send_msg_mock.call_count == 2 assert send_msg_mock.call_args_list[0][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER @@ -3793,13 +3794,16 @@ def test_locked_pairs(default_conf_usdt, ticker_usdt, fee, exit_check=ExitCheckTuple(exit_type=ExitType.STOP_LOSS) ) trade.close(ticker_usdt_sell_down()['bid']) - assert freqtrade.strategy.is_pair_locked(trade.pair) + assert freqtrade.strategy.is_pair_locked(trade.pair, side='*') + # Boths sides are locked + assert freqtrade.strategy.is_pair_locked(trade.pair, side='long') + assert freqtrade.strategy.is_pair_locked(trade.pair, side='short') # reinit - should buy other pair. caplog.clear() freqtrade.enter_positions() - assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog) + assert log_has_re(fr"Pair {trade.pair} \* is locked.*", caplog) @pytest.mark.parametrize("is_short", [False, True]) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 801e0e35f..b66c12086 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -15,6 +15,7 @@ from freqtrade.enums import TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.persistence import LocalTrade, Order, Trade, clean_dry_run_db, init_db from freqtrade.persistence.migrations import get_last_sequence_ids, set_sequence_ids +from freqtrade.persistence.models import PairLock from tests.conftest import create_mock_trades, create_mock_trades_with_leverage, log_has, log_has_re @@ -1427,6 +1428,55 @@ def test_migrate_set_sequence_ids(): assert engine.begin.call_count == 0 +def test_migrate_pairlocks(mocker, default_conf, fee, caplog): + """ + Test Database migration (starting with new pairformat) + """ + caplog.set_level(logging.DEBUG) + # Always create all columns apart from the last! + create_table_old = """CREATE TABLE pairlocks ( + id INTEGER NOT NULL, + pair VARCHAR(25) NOT NULL, + reason VARCHAR(255), + lock_time DATETIME NOT NULL, + lock_end_time DATETIME NOT NULL, + active BOOLEAN NOT NULL, + PRIMARY KEY (id) + ) + """ + create_index1 = "CREATE INDEX ix_pairlocks_pair ON pairlocks (pair)" + create_index2 = "CREATE INDEX ix_pairlocks_lock_end_time ON pairlocks (lock_end_time)" + create_index3 = "CREATE INDEX ix_pairlocks_active ON pairlocks (active)" + insert_table_old = """INSERT INTO pairlocks ( + id, pair, reason, lock_time, lock_end_time, active) + VALUES (1, 'ETH/BTC', 'Auto lock', '2021-07-12 18:41:03', '2021-07-11 18:45:00', 1) + """ + insert_table_old2 = """INSERT INTO pairlocks ( + id, pair, reason, lock_time, lock_end_time, active) + VALUES (2, '*', 'Lock all', '2021-07-12 18:41:03', '2021-07-12 19:00:00', 1) + """ + engine = create_engine('sqlite://') + mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) + # Create table using the old format + with engine.begin() as connection: + connection.execute(text(create_table_old)) + + connection.execute(text(insert_table_old)) + connection.execute(text(insert_table_old2)) + connection.execute(text(create_index1)) + connection.execute(text(create_index2)) + connection.execute(text(create_index3)) + + init_db(default_conf['db_url'], default_conf['dry_run']) + + assert len(PairLock.query.all()) == 2 + assert len(PairLock.query.filter(PairLock.pair == '*').all()) == 1 + pairlocks = PairLock.query.filter(PairLock.pair == 'ETH/BTC').all() + assert len(pairlocks) == 1 + pairlocks[0].pair == 'ETH/BTC' + pairlocks[0].side == '*' + + def test_adjust_stop_loss(fee): trade = Trade( pair='ADA/USDT', diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 97f367608..65df2d84c 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -10,7 +10,8 @@ from plotly.subplots import make_subplots from freqtrade.commands import start_plot_dataframe, start_plot_profit from freqtrade.configuration import TimeRange from freqtrade.data import history -from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data +from freqtrade.data.btanalysis import load_backtest_data +from freqtrade.data.metrics import create_cum_profit from freqtrade.exceptions import OperationalException from freqtrade.plot.plotting import (add_areas, add_indicators, add_profit, create_plotconfig, generate_candlestick_graph, generate_plot_filename,